fix link to example in readme

.gitignore

@@ -1,2 +1 @@
-config.py
 __pycache__

README.md

@@ -7,21 +7,33 @@ Geminer is a tool that was originally designed to convert a PicoCMS blog into a
 ## Features
 
 * Markdown to Gemtext conversion
-* Custom conversion for local links (soon)
-* Custom metadata gathering
-* Custom indexes (for instance by tags, by authors, or any metadata you desire)
+* Conversion of local links
+* Give your own metadata list to gather
+* Custom indexes
 * Jinja2 templating
 
+## Workflow
+
+Geminer execution can be decomposed in two steps :
+
+1. Parse blog posts markdown files and write gemtext translation.
+2. Generate meta pages, i.e. home page and custom indexes.
+
+During the first step, frontmatter metadata is collected from markdown posts while gemtext posts are generated. This means that while rendering the template, a post will only have access to informations about itself.
+
+During the second step, all metadata has been gathered, which enables creation of various indexed, which requires of course access to all posts metadata.
+
 ## Configuration
 
-Soon. For now you can read [the example config](config.py.example).
+Soon. For now you can read [the example config](example/config.py).
 
 ## Gemini capsules using Geminer
 
-* [Hashtagueule](gemini://hashtagueule.fr)
+* gemini://hashtagueule.fr
 
 ## TODO
 
 * add parameter to give a function to treat local links
 * clean the code (lots of work)
+* add feed generation
 * change configuration format?

example/config.py

@@ -2,46 +2,45 @@
 # It is not intended to be executed.
 
 # locale (for templates, for example dates rendering)
-locale = "fr_FR.utf8"
+locale = "en_US.utf8"
 
 # path to directory containing markdonw files to convert
-md_dir = "/srv/my-site-content/posts"
+md_path = "/srv/gemini/example/md"
 
-# path to directory where gemini files will be exported
-gmi_content_dir = "/srv/gemini/my-site/posts"
+# path to gemini blog root directory
+gmi_path = "/srv/gemini/example/gmi"
 
-# path to directory where meta pages will be generated
-# such as home page, tags list, tag pages, author pages...
-meta_dir = "/srv/gemini/my-site"
+# directory within gmi_path which will contains converted posts
+posts_dir = "posts"
 
 # path to directory containing templates
-tpl_dir = "~/repo/htg-content/gemini/templates"
+tpl_path = "/srv/gemini/example/templates"
 
 # list of markdown files extensions
 # Any file with a different extension will be ignored.
 md_extensions = [
-        "markdown",
-        "mdown",
-        "mkdn",
-        "md",
-        "mkd",
-        "mdwn",
-        "mdtxt",
-        "mdtext",
-        "text",
-        "Rmd"
+        ".markdown",
+        ".mdown",
+        ".mkdn",
+        ".md",
+        ".mkd",
+        ".mdwn",
+        ".mdtxt",
+        ".mdtext",
+        ".text",
+        ".Rmd"
         ]
 
-# Specify gemini files extension. Don't specify to disable extension.
+# Specify gemini files extension. Set to empty string to disable extension.
 # Warning: disabling could have unwanted side effects.
 # Check out README for more informations.
-gmi_extension = "gmi"
+gmi_extension = ".gmi"
 
 # replacement map
 # Some CMS make you use some placeholders (for instance for assets URL).
 # You have to inform geminer of them here.
 replace = [
-        ("%assets_url%", "https://hashtagueule.fr/assets")
+        ("%assets_url%", "https://example.com/assets")
 ]
 
 # md2gemini settings
@@ -91,6 +90,7 @@ post_props = [
 #   * index_tpl (facultative):  template of the property values index
 #
 # Filenames are relative to meta_dir and extensions are automatically added.
+# If filename contains an extension, it will override gmi_extension value.
 # When a string value is facultative, it defaults to property name, except for
 # index_name, which disables property values global index if not specified.
 index_props = [
@@ -111,3 +111,22 @@ index_props = [
         "index_tpl": "authors_index"
     }
 ]
+
+
+# custom extra pages to generate
+# Each entry will generate a single page.
+# This is the place to define homepage and feed page for instance.
+# Templates will have to handle the full unsorted list of posts.
+# "name" key is mandatory. It is the filename of the page.
+# If filename contains an extension, it will override gmi_extension value.
+# "tpl" key is facultative, defaults to name (without extension if any)
+custom_pages = [
+    {
+        "name": "index",
+        "tpl": "index"
+    },
+    {
+        "name": "atom.xml",
+        "tpl": "atom"
+    }
+]

example/md/firt_post.md

@@ -0,0 +1,9 @@
+---
+title: "First post"
+author: raspbeguy
+date: 2020-12-09
+tags: foo,bar
+---
+# I am a post
+
+Look at me!

example/templates/atom.tpl

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>My blog</title>
+  <subtitle>Les gentils du net</subtitle>
+  <link href="gemini://example.com/atom.xml" rel="self"/>
+  <link href="gemini://example.com" rel="alternate"/>
+  <updated>{{ now().strftime('%FT%TZ') }}</updated>
+  <author>
+    <name>raspbeguy</name>
+  </author>
+  <category term="tech" />
+  <id>gemini://example.com</id>
+{%- for post in posts|sort(attribute="date",reverse=True) %}
+  <entry>
+    <title>{{ post.title }}</title>
+    <id>gemini://example.com/{{ post.path }}</id>
+    <link href="gemini://example.com/{{ post.path }}" rel="alternate"/>
+    <updated>{{ post.date.strftime('%FT%TZ') }}</updated>
+    <published>{{ post.date.strftime('%FT%TZ') }}</published>
+    <author>
+      <name>{{ post.author.name }}</name>
+    </author>
+{%- for tag in post.tags %}
+    <category term="{{ tag.name }}"/>
+{%- endfor %}
+  </entry>
+{%- endfor %}
+</feed>

example/templates/author.tpl

@@ -0,0 +1,5 @@
+# Posts by {{ prop_item.name }}
+
+{% for post in prop_item.posts|sort(attribute="date",reverse=True) -%}
+=> /{{ post.path }} [{{ post.date.strftime('%d/%m/%Y') }}] {{ post.title }}
+{% endfor %}

example/templates/authors_index.tpl

@@ -0,0 +1,5 @@
+# Authors list
+
+{% for author, value in prop.items()|sort(attribute='1.name') -%}
+=> /authors/{{ author }}.gmi {{ value.name }} ({{ value.posts | length }} articles)
+{% endfor %}

example/templates/index.tpl

@@ -0,0 +1,18 @@
+# My blog
+
+## I deserve all your attention.
+
+This is my blog. Please read it and like it.
+
+Last posts :
+{% for post in (posts|sort(attribute="date",reverse=True))[:30] -%}
+=> /{{ post.path }} {{ post.date.strftime('%Y-%m-%d') }} - {{ post.title }}
+{% endfor %}
+=> /posts.gmi All posts
+=> /tags.gmi Tag list
+=> /authors.gmi Authors list
+
+=> /atom.xml Atom feed
+
+This capsule has been generated by Geminer.
+=> https://git.gugod.fr/raspbeguy/geminer Dépôt de Geminer

example/templates/post.tpl

@@ -0,0 +1,14 @@
+# {{ post.title }}
+
+by {{ post.author.name }}, on {{ post.date.strftime('%d %B %Y') }}
+{% set tags = [] %}{% for tag in post.tags %}{{ tags.append(tag.name) or "" }}{% endfor %}{{ tags|join(", ") }}
+
+{{ post.content }}
+
+=> / Back to home
+=> /authors/{{ post.author.slug }}.gmi More posts by {{ post.author.name }}
+
+Posts having those tags:
+{% for tag in post.tags|sort(attribute="name") -%}
+=> /tags/{{ tag.slug }}.gmi {{ tag.name }}
+{% endfor %}

example/templates/posts_list.tpl

@@ -0,0 +1,5 @@
+# Posts list
+
+{% for post in posts|sort(attribute="date",reverse=True) -%}
+=> /{{ post.path }} [{{ post.date.strftime('%d/%m/%Y') }}] {{ post.title }}
+{% endfor %}

example/templates/tag.tpl

@@ -0,0 +1,5 @@
+# Posts with tag "{{ prop_item.name }}"
+
+{% for post in prop_item.posts|sort(attribute="date",reverse=True) -%}
+=> /{{ post.path }} [{{ post.date.strftime('%d/%m/%Y') }}] {{ post.title }}
+{% endfor %}

example/templates/tags_index.tpl

@@ -0,0 +1,5 @@
+# Tags list
+
+{% for tag, value in prop.items()|sort(attribute='1.name') -%}
+=> /tags/{{ tag }}.gmi {{ value.name }} ({{ value.posts | length }} articles)
+{% endfor %}

geminer.py

@@ -13,62 +13,60 @@ import config
 # locale (for templates, for example dates rendering)
 locale.setlocale(locale.LC_ALL, config.locale)
 
-md_path = os.path.expanduser(config.md_dir)
-gmi_path = os.path.expanduser(config.gmi_dir)
-tpl_path = os.path.expanduser(config.tpl_dir)
-meta_path = os.path.expanduser(config.meta_dir)
+md_path = os.path.abspath(os.path.expanduser(config.md_path))
+gmi_path = os.path.abspath(os.path.expanduser(config.gmi_path))
+tpl_path = os.path.abspath(os.path.expanduser(config.tpl_path))
+
+posts_path = os.path.abspath(gmi_path + "/" + config.posts_dir)
 
 # Initiate meta lists
-posts = []
-posts_prop_index = {}
+posts = [] # This is a flat, unsorted list of posts
+posts_prop_index = {} # This is a dict containing posts sorted by properties
+
 for prop_dict in config.index_props:
     posts_prop_index[prop_dict["property"]] = {}
 
-os.chdir(md_path)
 
 def add_ext_gmi(link):
     # Custom function to apply to links
-    if "://" not in link: # apply only on local links
-        return link+".gmi"
+    if "://" not in link:  # apply only on local links
+        return os.path.splitext(link)[0] + ".gmi"
     else:
         return link
 
+
 # Walk through markdown directories
-for dirname, subdirlist, mdlist in os.walk('.'):
-    
+for dirname, subdirlist, mdlist in os.walk(md_path):
+
     # Create same hierarchy in GMI directory
-    gmi_subpath = os.path.abspath(gmi_path+"/"+dirname)
+    gmi_subpath = os.path.abspath(posts_path + "/" + os.path.relpath(dirname, md_path))
     os.makedirs(gmi_subpath, exist_ok=True)
 
     for mdfile in mdlist:
         basename, extension = os.path.splitext(mdfile)
-        extension = extension[1:]
-
-        post = {}
-
-        gmifile = basename
-        if config.gmi_extension:
-            gmifile += ".gmi"
-
-        # We need the relative path without the "./"
-        simpledirname = dirname[2:]
-        if simpledirname == "":
-            post["path"] = gmifile
-        else:
-            post["path"] = simpledirname + "/" + gmifile       
 
         # We want to ignore the file if this isn't a markdown file
         if extension not in config.md_extensions:
-            print("Ignoring file {}: \"{}\" not in markdown extensions list".format(mdfile, extension))
+            print(
+                'Ignoring file {}: "{}" not in markdown extensions list'.format(
+                    mdfile, extension
+                )
+            )
             pass
-        
+
+        post = {}
+
+        gmifile = basename + config.gmi_extension
+
+        post["path"] = config.posts_dir + "/" + os.path.relpath(dirname + "/" + gmifile, md_path)
+
         # Read the Markdown file
-        with open(dirname+"/"+mdfile, 'r') as md:
+        with open(dirname + "/" + mdfile, "r") as md:
             mdtext = md.read()
-        
+
         # Parse the YAML header
         meta = frontmatter.parse(mdtext)[0]
-       
+
         # Extract template
         post["template"] = meta.get("template", config.default_post_template)
 
@@ -81,20 +79,29 @@ for dirname, subdirlist, mdlist in os.walk('.'):
             prop = prop_dict["property"]
             prop_raw = meta.get(prop, None)
             if prop_dict.get("list", False) and prop_raw:
-                post[prop] = [{"name": word, "slug": slugify(word)} for word in prop_raw.split(',')]
+                post[prop] = [
+                    {"name": word, "slug": slugify(word)}
+                    for word in prop_raw.split(",")
+                ]
                 for item in post[prop]:
                     slug = item["slug"]
                     if slug in posts_prop_index[prop]:
                         posts_prop_index[prop][slug]["posts"].append(post)
                     else:
-                        posts_prop_index[prop][slug] = {"name": item["name"], "posts": [post]}
+                        posts_prop_index[prop][slug] = {
+                            "name": item["name"],
+                            "posts": [post],
+                        }
             else:
                 post[prop] = {"name": prop_raw, "slug": slugify(prop_raw)}
                 slug = post[prop]["slug"]
                 if slug in posts_prop_index[prop]:
                     posts_prop_index[prop][slug]["posts"].append(post)
                 else:
-                    posts_prop_index[prop][slug] = {"name": post[prop]["name"], "posts": [post]}
+                    posts_prop_index[prop][slug] = {
+                        "name": post[prop]["name"],
+                        "posts": [post],
+                    }
 
         posts.append(post)
 
@@ -103,65 +110,76 @@ for dirname, subdirlist, mdlist in os.walk('.'):
 
         # Replace stuff
         for item in config.replace:
-            mdtext = mdtext.replace(item[0],item[1])
+            mdtext = mdtext.replace(item[0], item[1])
 
         # Convert the post into GMI
-        gmitext = md2gemini(mdtext,
-                code_tag=config.code_tag,
-                img_tag=config.img_tag,
-                indent=config.indent,
-                ascii_table=config.ascii_table,
-                frontmatter=True,
-                links=config.links,
-                plain=config.plain,
-                strip_html=config.strip_html,
-                base_url=config.base_url,
-                link_func=add_ext_gmi,
-                table_tag=config.table_tag
-                )
+        gmitext = md2gemini(
+            mdtext,
+            code_tag=config.code_tag,
+            img_tag=config.img_tag,
+            indent=config.indent,
+            ascii_table=config.ascii_table,
+            frontmatter=True,
+            links=config.links,
+            plain=config.plain,
+            strip_html=config.strip_html,
+            base_url=config.base_url,
+            link_func=add_ext_gmi,
+            table_tag=config.table_tag,
+        )
+
+        post["content"] = gmitext
 
         # Read template file
-        with open(tpl_path+"/"+post["template"]+".tpl", 'r') as tpl:
+        with open(tpl_path + "/" + post["template"] + ".tpl", "r") as tpl:
             template = Template(tpl.read())
 
         # Integrate the GMI content in the template
-        gmitext = template.render(content=gmitext, meta = post)
-        
+        gmitext = template.render(post=post)
+
         # Dirty fix a weird bug where some lines are CRLF-terminated
-        gmitext = gmitext.replace('\r\n','\n')
-        
+        gmitext = gmitext.replace("\r\n", "\n")
 
         # Time to write the GMI file
-        with open(gmi_subpath+"/"+gmifile, 'w') as gmi:
+        with open(gmi_subpath + "/" + gmifile, "w") as gmi:
             gmi.write(gmitext)
 
-# Generate home page
-with open(tpl_path+"/index.tpl", 'r') as tpl:
-    template = Template(tpl.read())
-text = template.render(posts=posts)
-with open(meta_path+"/index.gmi", 'w') as gmi:
-    gmi.write(text)
+# Generate custom extra pages
+for page_dict in config.custom_pages:
+    rel_path, filename = os.path.split(page_dict["name"])
+    if rel_path:
+        os.makedirs(rel_path, exist_ok=True)
+    basename, extension = os.path.splitext(filename)
+    if extension == "":
+        filename = basename + config.gmi_extension
+    else:
+        filename = basename + extension
+    filepath = os.path.join(rel_path, filename)
+    with open(tpl_path + "/" + page_dict.get("tpl", basename) + ".tpl", "r") as tpl:
+        template = Template(tpl.read())
+    template.globals['now'] = datetime.now
+    text = template.render(posts=posts)
+    text = text.replace("\r\n", "\n")
+    with open(gmi_path + "/" + filepath, "w") as gmi:
+        gmi.write(text)
 
 # Generate custom meta pages
 for prop_dict in config.index_props:
     prop = prop_dict["property"]
     if "index_name" in prop_dict:
-        with open(tpl_path+"/"+prop_dict.get("index_tpl",prop)+".tpl", 'r') as tpl:
+        with open(
+            tpl_path + "/" + prop_dict.get("index_tpl", prop) + ".tpl", "r"
+        ) as tpl:
             template = Template(tpl.read())
         text = template.render(prop=posts_prop_index[prop])
-        with open(meta_path+"/"+prop_dict["index_name"]+'.gmi', 'w') as gmi:
+        with open(gmi_path + "/" + prop_dict["index_name"] + config.gmi_extension, "w") as gmi:
             gmi.write(text)
-    os.makedirs(meta_path+"/"+prop_dict.get("item_dir", prop), exist_ok=True)
-    with open(tpl_path+"/"+prop_dict.get("item_tpl", prop)+".tpl", 'r') as tpl:
+    os.makedirs(gmi_path + "/" + prop_dict.get("item_dir", prop), exist_ok=True)
+    with open(tpl_path + "/" + prop_dict.get("item_tpl", prop) + ".tpl", "r") as tpl:
         template = Template(tpl.read())
     for item in posts_prop_index[prop]:
         text = template.render(prop_item=posts_prop_index[prop][item])
-        with open(meta_path+"/"+prop_dict.get("item_dir", prop)+"/"+item+".gmi", "w") as gmi:
+        with open(
+            gmi_path + "/" + prop_dict.get("item_dir", prop) + "/" + item + config.gmi_extension, "w"
+        ) as gmi:
             gmi.write(text)
-
-# Generate posts list page
-with open(tpl_path+"/posts_list.tpl", 'r') as tpl:
-    template = Template(tpl.read())
-text = template.render(posts=posts)
-with open(meta_path+"/posts.gmi", 'w') as gmi:
-    gmi.write(text)