fix link to example in readme

Reviewed-on: #1

.gitignore

@@ -1,2 +1 @@
-config.py
 __pycache__

README.md

@@ -0,0 +1,39 @@
+# Geminer
+
+## Introduction
+
+Geminer is a tool that was originally designed to convert a PicoCMS blog into a static version for Gemini. In fact, it can act as a markdown-based static site generator.
+
+## Features
+
+* Markdown to Gemtext conversion
+* Conversion of local links
+* Give your own metadata list to gather
+* Custom indexes
+* Jinja2 templating
+
+## Workflow
+
+Geminer execution can be decomposed in two steps :
+
+1. Parse blog posts markdown files and write gemtext translation.
+2. Generate meta pages, i.e. home page and custom indexes.
+
+During the first step, frontmatter metadata is collected from markdown posts while gemtext posts are generated. This means that while rendering the template, a post will only have access to informations about itself.
+
+During the second step, all metadata has been gathered, which enables creation of various indexed, which requires of course access to all posts metadata.
+
+## Configuration
+
+Soon. For now you can read [the example config](example/config.py).
+
+## Gemini capsules using Geminer
+
+* gemini://hashtagueule.fr
+
+## TODO
+
+* add parameter to give a function to treat local links
+* clean the code (lots of work)
+* add feed generation
+* change configuration format?

config.py.example

@@ -1,51 +0,0 @@
-# This is the configuration file for geminer.
-# It is not intended to be executed.
-
-# locale (for templates, for example dates rendering)
-locale = "fr_FR.utf8"
-
-# path to directory containing markdonw files to convert
-md_dir = "~/repo/htg-content/content/posts"
-
-# path to directory where gemini files will be exported
-gmi_dir = "/tmp/gemini"
-
-# path to directory containing templates
-tpl_dir = "~/repo/htg-content/gemini/templates"
-
-# list of markdown files extensions
-# Any file with a different extension will be ignored.
-md_extensions = [
-        "markdown",
-        "mdown",
-        "mkdn",
-        "md",
-        "mkd",
-        "mdwn",
-        "mdtxt",
-        "mdtext",
-        "text",
-        "Rmd"
-        ]
-
-# Set following value to True if you want ".gmi" extensions on new files.
-gmi_extension = False
-
-# replacement map
-# Some CMS make you use some placeholders (for instance for assets URL).
-# You have to inform geminer of them here.
-replace = [
-        ("%assets_url%", "https://hashtagueule.fr/assets")
-        ]
-
-# md2gemini settings
-# Check the documentation at https://pypi.org/project/md2gemini/
-code_tag=""
-img_tag="[IMG]"
-indent=" "
-ascii_table=False
-links="copy"
-plain=True
-strip_html=False
-base_url=""
-table_tag="table"

example/config.py

@@ -0,0 +1,132 @@
+# This is the configuration file for geminer.
+# It is not intended to be executed.
+
+# locale (for templates, for example dates rendering)
+locale = "en_US.utf8"
+
+# path to directory containing markdonw files to convert
+md_path = "/srv/gemini/example/md"
+
+# path to gemini blog root directory
+gmi_path = "/srv/gemini/example/gmi"
+
+# directory within gmi_path which will contains converted posts
+posts_dir = "posts"
+
+# path to directory containing templates
+tpl_path = "/srv/gemini/example/templates"
+
+# list of markdown files extensions
+# Any file with a different extension will be ignored.
+md_extensions = [
+        ".markdown",
+        ".mdown",
+        ".mkdn",
+        ".md",
+        ".mkd",
+        ".mdwn",
+        ".mdtxt",
+        ".mdtext",
+        ".text",
+        ".Rmd"
+        ]
+
+# Specify gemini files extension. Set to empty string to disable extension.
+# Warning: disabling could have unwanted side effects.
+# Check out README for more informations.
+gmi_extension = ".gmi"
+
+# replacement map
+# Some CMS make you use some placeholders (for instance for assets URL).
+# You have to inform geminer of them here.
+replace = [
+        ("%assets_url%", "https://example.com/assets")
+]
+
+# md2gemini settings
+# Check the documentation at https://pypi.org/project/md2gemini/l
+code_tag=""
+img_tag="[IMG]"
+indent=" "
+ascii_table=False
+links="copy"
+plain=True
+strip_html=False
+base_url=""
+table_tag="table"
+
+# default template for posts that don't specify one
+default_post_template = "post"
+
+# per-post properties to fetch in frontmatter
+post_props = [
+    "date",
+    "title"
+    ]
+
+# indexable properties to fetch in frontmatter
+# A lot of CMS will manage properties like tags ans authors, which are written
+# in the frontmatter, and are used to make subgroups of posts.
+# This setting enables to do the same for any frontmatter property you wish.
+#
+# Each indexable property generate two views:
+#   * per-value post index:  This is a list of links to posts that have a given
+#                            value of the property. There are as many per-value
+#                            indexes as properties values.
+#   * property values index: This is a list of links to per-value indexes.
+#                            There is only one page of this kind, and it can be
+#                            disabled.
+#
+# Exemple: if the property is tags, then it will generate a tag index tags.gmi,
+# which will link to some subindexes like tags/computer (assuming computer is
+# an existing tag).
+#
+# Specify here a list of dictionnary, each one with the following keys:
+#   * property (mandatory):     name of the property present in the frontmatter
+#   * list (facultative):       set to True if the property is a list of values
+#   * item_dir (facultative):   directory containing per-value posts indexes
+#   * item_tpl (facultative):   template of the per-value posts indexes
+#   * index_name (facultative): filename of the property values index
+#   * index_tpl (facultative):  template of the property values index
+#
+# Filenames are relative to meta_dir and extensions are automatically added.
+# If filename contains an extension, it will override gmi_extension value.
+# When a string value is facultative, it defaults to property name, except for
+# index_name, which disables property values global index if not specified.
+index_props = [
+    {
+        "property": "tags",
+        "list": True,
+        "item_dir": "tags",
+        "item_tpl": "tag",
+        "index_name": "tags",
+        "index_tpl": "tags_index"
+    },
+    {
+        "property": "author",
+        "list": False,
+        "item_dir": "authors",
+        "item_tpl": "author",
+        "index_name": "authors",
+        "index_tpl": "authors_index"
+    }
+]
+
+
+# custom extra pages to generate
+# Each entry will generate a single page.
+# This is the place to define homepage and feed page for instance.
+# Templates will have to handle the full unsorted list of posts.
+# "name" key is mandatory. It is the filename of the page.
+# If filename contains an extension, it will override gmi_extension value.
+# "tpl" key is facultative, defaults to name (without extension if any)
+custom_pages = [
+    {
+        "name": "index",
+        "tpl": "index"
+    },
+    {
+        "name": "atom.xml",
+        "tpl": "atom"
+    }
+]

example/md/firt_post.md

@@ -0,0 +1,9 @@
+---
+title: "First post"
+author: raspbeguy
+date: 2020-12-09
+tags: foo,bar
+---
+# I am a post
+
+Look at me!

example/templates/atom.tpl

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>My blog</title>
+  <subtitle>Les gentils du net</subtitle>
+  <link href="gemini://example.com/atom.xml" rel="self"/>
+  <link href="gemini://example.com" rel="alternate"/>
+  <updated>{{ now().strftime('%FT%TZ') }}</updated>
+  <author>
+    <name>raspbeguy</name>
+  </author>
+  <category term="tech" />
+  <id>gemini://example.com</id>
+{%- for post in posts|sort(attribute="date",reverse=True) %}
+  <entry>
+    <title>{{ post.title }}</title>
+    <id>gemini://example.com/{{ post.path }}</id>
+    <link href="gemini://example.com/{{ post.path }}" rel="alternate"/>
+    <updated>{{ post.date.strftime('%FT%TZ') }}</updated>
+    <published>{{ post.date.strftime('%FT%TZ') }}</published>
+    <author>
+      <name>{{ post.author.name }}</name>
+    </author>
+{%- for tag in post.tags %}
+    <category term="{{ tag.name }}"/>
+{%- endfor %}
+  </entry>
+{%- endfor %}
+</feed>

example/templates/author.tpl

@@ -0,0 +1,5 @@
+# Posts by {{ prop_item.name }}
+
+{% for post in prop_item.posts|sort(attribute="date",reverse=True) -%}
+=> /{{ post.path }} [{{ post.date.strftime('%d/%m/%Y') }}] {{ post.title }}
+{% endfor %}

example/templates/authors_index.tpl

@@ -0,0 +1,5 @@
+# Authors list
+
+{% for author, value in prop.items()|sort(attribute='1.name') -%}
+=> /authors/{{ author }}.gmi {{ value.name }} ({{ value.posts | length }} articles)
+{% endfor %}

example/templates/index.tpl

@@ -0,0 +1,18 @@
+# My blog
+
+## I deserve all your attention.
+
+This is my blog. Please read it and like it.
+
+Last posts :
+{% for post in (posts|sort(attribute="date",reverse=True))[:30] -%}
+=> /{{ post.path }} {{ post.date.strftime('%Y-%m-%d') }} - {{ post.title }}
+{% endfor %}
+=> /posts.gmi All posts
+=> /tags.gmi Tag list
+=> /authors.gmi Authors list
+
+=> /atom.xml Atom feed
+
+This capsule has been generated by Geminer.
+=> https://git.gugod.fr/raspbeguy/geminer Dépôt de Geminer

example/templates/post.tpl

@@ -0,0 +1,14 @@
+# {{ post.title }}
+
+by {{ post.author.name }}, on {{ post.date.strftime('%d %B %Y') }}
+{% set tags = [] %}{% for tag in post.tags %}{{ tags.append(tag.name) or "" }}{% endfor %}{{ tags|join(", ") }}
+
+{{ post.content }}
+
+=> / Back to home
+=> /authors/{{ post.author.slug }}.gmi More posts by {{ post.author.name }}
+
+Posts having those tags:
+{% for tag in post.tags|sort(attribute="name") -%}
+=> /tags/{{ tag.slug }}.gmi {{ tag.name }}
+{% endfor %}

example/templates/posts_list.tpl

@@ -0,0 +1,5 @@
+# Posts list
+
+{% for post in posts|sort(attribute="date",reverse=True) -%}
+=> /{{ post.path }} [{{ post.date.strftime('%d/%m/%Y') }}] {{ post.title }}
+{% endfor %}

example/templates/tag.tpl

@@ -0,0 +1,5 @@
+# Posts with tag "{{ prop_item.name }}"
+
+{% for post in prop_item.posts|sort(attribute="date",reverse=True) -%}
+=> /{{ post.path }} [{{ post.date.strftime('%d/%m/%Y') }}] {{ post.title }}
+{% endfor %}

example/templates/tags_index.tpl

@@ -0,0 +1,5 @@
+# Tags list
+
+{% for tag, value in prop.items()|sort(attribute='1.name') -%}
+=> /tags/{{ tag }}.gmi {{ value.name }} ({{ value.posts | length }} articles)
+{% endfor %}

geminer.py

@@ -2,57 +2,119 @@
 
 from md2gemini import md2gemini
 import frontmatter
+from slugify import slugify
 from jinja2 import Template
 import os
 import locale
+from datetime import datetime
 
 import config
 
 # locale (for templates, for example dates rendering)
-locale = "fr_FR.utf8"
+locale.setlocale(locale.LC_ALL, config.locale)
 
-md_path = os.path.expanduser(config.md_dir)
+md_path = os.path.abspath(os.path.expanduser(config.md_path))
-gmi_path = os.path.expanduser(config.gmi_dir)
+gmi_path = os.path.abspath(os.path.expanduser(config.gmi_path))
-tpl_path = os.path.expanduser(config.tpl_dir)
+tpl_path = os.path.abspath(os.path.expanduser(config.tpl_path))
+
+posts_path = os.path.abspath(gmi_path + "/" + config.posts_dir)
+
+# Initiate meta lists
+posts = [] # This is a flat, unsorted list of posts
+posts_prop_index = {} # This is a dict containing posts sorted by properties
+
+for prop_dict in config.index_props:
+    posts_prop_index[prop_dict["property"]] = {}
+
+
+def add_ext_gmi(link):
+    # Custom function to apply to links
+    if "://" not in link:  # apply only on local links
+        return os.path.splitext(link)[0] + ".gmi"
+    else:
+        return link
 
-os.chdir(md_path)
 
 # Walk through markdown directories
-for dirname, subdirlist, mdlist in os.walk('.'):
+for dirname, subdirlist, mdlist in os.walk(md_path):
 
     # Create same hierarchy in GMI directory
-    gmi_subpath = os.path.abspath(gmi_path+"/"+dirname)
+    gmi_subpath = os.path.abspath(posts_path + "/" + os.path.relpath(dirname, md_path))
     os.makedirs(gmi_subpath, exist_ok=True)
 
     for mdfile in mdlist:
         basename, extension = os.path.splitext(mdfile)
-        extension = extension[1:]
 
         # We want to ignore the file if this isn't a markdown file
         if extension not in config.md_extensions:
-            print("Ignoring file {}: \"{}\" not in markdown extensions list".format(mdfile, extension))
+            print(
+                'Ignoring file {}: "{}" not in markdown extensions list'.format(
+                    mdfile, extension
+                )
+            )
             pass
 
+        post = {}
+
+        gmifile = basename + config.gmi_extension
+
+        post["path"] = config.posts_dir + "/" + os.path.relpath(dirname + "/" + gmifile, md_path)
+
         # Read the Markdown file
-        with open(dirname+"/"+mdfile, 'r') as md:
+        with open(dirname + "/" + mdfile, "r") as md:
             mdtext = md.read()
 
         # Parse the YAML header
         meta = frontmatter.parse(mdtext)[0]
 
-        # Extract useful informations from the header
+        # Extract template
-        template = meta.get("template", None)
+        post["template"] = meta.get("template", config.default_post_template)
-        author = meta.get("author", None)
+
-        date = meta.get("date", None)
+        # Extract post properties
-        title = meta.get("title", None)
+        for prop in config.post_props:
-        tags = meta.get("tags", None)
+            post[prop] = meta.get(prop, None)
-        # For now, tags list must be a comma-separated string
+
-        # TODO: make possible to list tags as a YAML list
+        # Extract index properties
+        for prop_dict in config.index_props:
+            prop = prop_dict["property"]
+            prop_raw = meta.get(prop, None)
+            if prop_dict.get("list", False) and prop_raw:
+                post[prop] = [
+                    {"name": word, "slug": slugify(word)}
+                    for word in prop_raw.split(",")
+                ]
+                for item in post[prop]:
+                    slug = item["slug"]
+                    if slug in posts_prop_index[prop]:
+                        posts_prop_index[prop][slug]["posts"].append(post)
+                    else:
+                        posts_prop_index[prop][slug] = {
+                            "name": item["name"],
+                            "posts": [post],
+                        }
+            else:
+                post[prop] = {"name": prop_raw, "slug": slugify(prop_raw)}
+                slug = post[prop]["slug"]
+                if slug in posts_prop_index[prop]:
+                    posts_prop_index[prop][slug]["posts"].append(post)
+                else:
+                    posts_prop_index[prop][slug] = {
+                        "name": post[prop]["name"],
+                        "posts": [post],
+                    }
+
+        posts.append(post)
+
+        # For now, list properties must be comma-separated strings.
+        # TODO: make possible to list values as a YAML list
+
+        # Replace stuff
         for item in config.replace:
             mdtext = mdtext.replace(item[0], item[1])
 
         # Convert the post into GMI
-        gmitext = md2gemini(mdtext,
+        gmitext = md2gemini(
+            mdtext,
             code_tag=config.code_tag,
             img_tag=config.img_tag,
             indent=config.indent,
@@ -62,39 +124,62 @@ for dirname, subdirlist, mdlist in os.walk('.'):
             plain=config.plain,
             strip_html=config.strip_html,
             base_url=config.base_url,
-                md_links=True,
+            link_func=add_ext_gmi,
-                table_tag=config.table_tag
+            table_tag=config.table_tag,
         )
 
+        post["content"] = gmitext
+
         # Read template file
-        with open(tpl_path+"/"+template+".tpl", 'r') as tpl:
+        with open(tpl_path + "/" + post["template"] + ".tpl", "r") as tpl:
             template = Template(tpl.read())
 
-        # We need the relative path without the "./"
-        simpledirname = dirname[2:]
-        if simpledirname == "":
-            path = basename
-        else:
-            path = simpledirname + "/" + basename
-        
         # Integrate the GMI content in the template
-        gmitext = template.render(
+        gmitext = template.render(post=post)
-                content=gmitext,
-                tags=tags,
-                template=template,
-                author=author,
-                date=date,
-                title=title,
-                path=path
-                )
 
         # Dirty fix a weird bug where some lines are CRLF-terminated
-        gmitext = gmitext.replace('\r\n','\n')
+        gmitext = gmitext.replace("\r\n", "\n")
 
         # Time to write the GMI file
-        gmifile = basename
+        with open(gmi_subpath + "/" + gmifile, "w") as gmi:
-        if config.gmi_extension:
-            gmifile += ".gmi"
-        print(gmi_subpath+"/"+gmifile)
-        with open(gmi_subpath+"/"+gmifile, 'w') as gmi:
             gmi.write(gmitext)
+
+# Generate custom extra pages
+for page_dict in config.custom_pages:
+    rel_path, filename = os.path.split(page_dict["name"])
+    if rel_path:
+        os.makedirs(rel_path, exist_ok=True)
+    basename, extension = os.path.splitext(filename)
+    if extension == "":
+        filename = basename + config.gmi_extension
+    else:
+        filename = basename + extension
+    filepath = os.path.join(rel_path, filename)
+    with open(tpl_path + "/" + page_dict.get("tpl", basename) + ".tpl", "r") as tpl:
+        template = Template(tpl.read())
+    template.globals['now'] = datetime.now
+    text = template.render(posts=posts)
+    text = text.replace("\r\n", "\n")
+    with open(gmi_path + "/" + filepath, "w") as gmi:
+        gmi.write(text)
+
+# Generate custom meta pages
+for prop_dict in config.index_props:
+    prop = prop_dict["property"]
+    if "index_name" in prop_dict:
+        with open(
+            tpl_path + "/" + prop_dict.get("index_tpl", prop) + ".tpl", "r"
+        ) as tpl:
+            template = Template(tpl.read())
+        text = template.render(prop=posts_prop_index[prop])
+        with open(gmi_path + "/" + prop_dict["index_name"] + config.gmi_extension, "w") as gmi:
+            gmi.write(text)
+    os.makedirs(gmi_path + "/" + prop_dict.get("item_dir", prop), exist_ok=True)
+    with open(tpl_path + "/" + prop_dict.get("item_tpl", prop) + ".tpl", "r") as tpl:
+        template = Template(tpl.read())
+    for item in posts_prop_index[prop]:
+        text = template.render(prop_item=posts_prop_index[prop][item])
+        with open(
+            gmi_path + "/" + prop_dict.get("item_dir", prop) + "/" + item + config.gmi_extension, "w"
+        ) as gmi:
+            gmi.write(text)

requirements.txt

@@ -0,0 +1,4 @@
+Jinja2
+md2gemini
+python-frontmatter
+python-slugify