Display a calendar heat map using Emacs Lisp

| emacs

I was curious about how to quickly visualize my date-related data in Emacs, such as when I sketched my thoughts or which days had journal entries or how often A- had tantrums. (It's hard to be 6 years old.) I wrote this code based on nrougier's code for colouring calendar days using advice around calendar-generate-entries:

(defface calendar-scale-1  '((((background light)) :foreground "black" :background "#eceff1")
                             (((background dark))  :foreground "white" :background "#263238")) "")
(defface calendar-scale-2  '((((background light)) :foreground "black" :background "#cfd8dc")
                             (((background dark))  :foreground "white" :background "#37474f")) "")
(defface calendar-scale-3  '((((background light)) :foreground "black" :background "#b0bec5")
                             (((background dark))  :foreground "white" :background "#455a64")) "")
(defface calendar-scale-4  '((((background light)) :foreground "black" :background "#90a4ae")
                             (((background dark))  :foreground "white" :background "#546e7a")) "")
(defface calendar-scale-5  '((((background light)) :foreground "black" :background "#78909c")
                             (((background dark))  :foreground "white" :background "#607d8b")) "")
(defface calendar-scale-6  '((((background light)) :foreground "black" :background "#607d8b")
                             (((background dark))  :foreground "white" :background "#78909c")) "")
(defface calendar-scale-7  '((((background light)) :foreground "black" :background "#546e7a")
                             (((background dark))  :foreground "white" :background "#90a4ae")) "")
(defface calendar-scale-8  '((((background light)) :foreground "black" :background "#455a64")
                             (((background dark))  :foreground "white" :background "#b0bec5")) "")
(defface calendar-scale-9  '((((background light)) :foreground "black" :background "#37474f")
                             (((background dark))  :foreground "white" :background "#cfd8dc")) "")
(defface calendar-scale-10 '((((background light)) :foreground "black" :background "#263238")
                             (((background dark))  :foreground "white" :background "#eceff1")) "")

(defun my-count-calendar-entries (grouped-entries)
  (mapcar (lambda (entry) (cons (car entry) (length (cdr entry)))) grouped-entries))

(defun my-scale-calendar-entries (grouped-entries &optional scale-max)
  (let* ((count (my-count-calendar-entries grouped-entries))
         (count-max (apply #'max (mapcar (lambda (o) (if (car o) (cdr o) 0)) count))))
    (mapcar (lambda (entry)
              (cons (car entry)
                    (/ (* 1.0 (or scale-max 1.0) (cdr entry)) count-max)))
            count)))

(defun my-scale-calendar-entries-logarithmically (grouped-entries &optional scale-max)
  (let* ((count (my-count-calendar-entries grouped-entries))
         (count-max (apply #'max (mapcar (lambda (o) (if (car o) (cdr o) 0)) count))))
    (mapcar (lambda (entry)
              (cons (car entry)
                    (/ (* 1.0 (or scale-max 1.0) (log (cdr entry))) (log count-max))))
            count)))

(defvar my-calendar-count-scaled nil "Values to display.")

(defun my-calendar-heat-map (month year indent)
  (when my-calendar-count-scaled
    (dotimes (i 31)
      (let ((date (list month (1+ i) year))
            (count-scaled (assoc-default (format "%04d-%02d-%02d" year month (1+ i))
                                         my-calendar-count-scaled)))
        (when count-scaled
          (calendar-mark-visible-date
           date
           (intern (format "calendar-scale-%d" count-scaled))))))))

(advice-add #'calendar-generate-month :after #'my-calendar-heat-map)
;(advice-remove #'calendar-generate-month #'my-calendar-heat-map)

(defun my-calendar-visualize (values)
  (setq my-calendar-count-scaled values)
  (calendar))

Journal entries

So if I want to visualize the days with journal entries, I can use this code:

(defun my-calendar-visualize-journal-entries ()
  (interactive)
  (my-calendar-visualize
   (mapcar
    (lambda (o)
      (cons
       (car o)
       (ceiling (+ 1 (* 7.0 (cdr o))))))
    (my-scale-calendar-entries
     (seq-group-by #'my-journal-date
                   (cdr (pcsv-parse-file "~/Downloads/entries.csv")))))))
2023-01-05_21-42-06.png
Figure 1: Journal entries

Sketches

(defun my-calendar-visualize-sketches ()
  (interactive)
  (let ((my-calendar-sketches
         (assoc-delete-all
          nil
          (seq-group-by
           (lambda (o)
             (when (string-match "^\\([0-9][0-9][0-9][0-9]\\)[-_]?\\([0-9][0-9]\\)[-_]?\\([0-9][0-9]\\)" o)
               (format "%s-%s-%s"
                       (match-string 1 o)
                       (match-string 2 o)
                       (match-string 3 o))))
           (append
            (directory-files "~/sync/sketches" nil "\\.\\(png\\|jpg\\)\\'")
            (directory-files "~/sync/private-sketches" nil "\\.\\(png\\|jpg\\)\\'"))))))
    (my-calendar-visualize
     (mapcar
      (lambda (o)
        (cons (car o)
              ;; many days have just 1 sketch, so I set the low end of the scale
              ;; to make them visible, and use a logarithmic scale for the rest
              (ceiling (+ 3 (* 7.0 (cdr o))))))
      (my-scale-calendar-entries-logarithmically my-calendar-sketches)))))
2023-01-05_21-37-03.png
Figure 2: Sketches

Big feelings

(defun my-calendar-visualize-tantrums ()
  (interactive)
  (my-calendar-visualize
   (mapcar
    (lambda (o)
      (cons
       (car o)
       (ceiling (* 10.0 (cdr o)))))
    (my-scale-calendar-entries
     (seq-group-by #'my-journal-date
                   (seq-filter (lambda (o) (string-match "tantrum\\|grump\\|angry\\|meltdown"
                                                           (my-journal-note o)))
                               (cdr (pcsv-parse-file "~/Downloads/entries.csv"))))))))
2023-01-05_21-46-22.png
Figure 3: Tantrums and meltdowns

(The start of the schoolyear was pretty rough.)

I'd like to figure out a yearly calendar view, and maybe use the calendar as a way to navigate my data too. calendar-mark-visible-date relies on the position and gets confused by the stuff I tried from these yearly calendar hacks, but maybe I can change calendar-date-echo-text to '(calendar-iso-date-string (list month day year)) and then extract the data from the help-echo property, since mysteriously, the date doesn't actually seem to be otherwise stored in the calendar. Anyway, I'll post that when I figure it out!

Rename, recolor, and file my sketches automatically

| geek, supernote, python, drawing

I want to make it easier to process the sketchnotes I make on my Supernote. I write IDs of the form yyyy-mm-dd-nn to identify my sketches. To avoid duplicates, I get these IDs from the web-based journaling system I wrote. I've started putting the titles and tags into those journal entries as well so that I can reuse them in scripts. When I export a sketch to PNG and synchronize it, the file appears in my ~/Dropbox/Supernote/EXPORT directory on my laptop. Then it goes through this process:

  • I use Google Cloud Vision to detect handwriting so that I can find the ID.
    • I retrieve the matching entry from my journal system and rename the file based on the title and tags.
    • If there's no matching entry, I rename the file based on the ID.
  • If there are other tags or references in the sketch, I add those to the filename as well.
  • I recolor it based on the tags, so parenting-related posts are a little purple, tech/Emacs-related posts are blue, and things are generally highlighted in yellow otherwise.
  • I move it to a directory based on the tags.
    • If it's a private sketch, I move it to the directory for my private sketches.
    • If it's a public sketch, I move it to the directory that will eventually get synchronized to sketches.sachachua.com, and I reload the list of sketches after some delay.

The following code does that processing.

Download supernote-daemon

supernote-daemon source code
#!/usr/bin/python3
# -*- mode: python -*-

# (c) 2022-2023 Sacha Chua (sacha@sachachua.com) - MIT License

# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


import os
import json
import re
import requests
import time
from dotenv import load_dotenv
# Import the Google Cloud client libraries
from google.cloud import vision
from google.cloud.vision_v1 import AnnotateImageResponse
import sys
sys.path.append("/home/sacha/proj/supernote/")
import recolor   # noqa: E402  # muffles flake8 error about import
load_dotenv()


# Set the folder path where the png files are located
folder_path = '/home/sacha/Dropbox/Supernote/EXPORT/'
public_sketch_dir = '/home/sacha/sync/sketches/'
private_sketch_dir = '/home/sacha/sync/private-sketches/'

# Initialize the Google Cloud Vision client
client = vision.ImageAnnotatorClient()
refresh_counter = 0

def extract_text(client, file):
    json_file = file[:-3] + 'json'
    # TODO Preprocess to keep only black text
    with open(file, 'rb') as image_file:
        content = image_file.read()
    # Convert the png file to a Google Cloud Vision image object
    image = vision.Image(content=content)

    # Extract handwriting from the image using the Google Cloud Vision API
    response = client.document_text_detection(image=image)
    response_json = AnnotateImageResponse.to_json(response)
    json_response = json.loads(response_json)
    # Save the response to a json file with the same name as the png file
    with open(json_file, "w") as f:
        json.dump(json_response, f)


def maybe_rename(file):
    # TODO Match on ID
    json_file = file[:-3] + 'json'
    with open(json_file, 'r') as f:
        data = json.load(f)

    # Extract the text from the json file
    text = data['fullTextAnnotation']['text']

    # Check if the text contains a string matching the regex pattern
    pattern = r'(?<!ref:)[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}'
    match = re.search(pattern, text)
    if match:
        # Get the matched string
        matched_string = match.group(0)
        new_name = matched_string
        from_zid = get_journal_entry(matched_string).strip()
        if from_zid:
            new_name = matched_string + ' ' + from_zid
        tags = get_tags(new_name, text)
        if tags:
            new_name = new_name + ' ' + tags
        ref = get_references(text)
        if ref:
            new_name = new_name + ' ' + ref
        print('Renaming ' + file + ' to ' + new_name)
        # Rename the png and json files to the matched string
        new_filename = os.path.join(os.path.dirname(file), new_name + '.png')
        rename_set(file, new_filename)
        return new_filename


def get_tags(filename, text):
    tags = re.findall(r'(^|\W)#[ \n\t]+', text)
    return ' '.join(filter(lambda x: x not in filename, tags))


def get_references(text):
    refs = re.findall(r'!ref:[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}', text)
    return ' '.join(refs)


def get_journal_entry(zid):
    resp = requests.get('https://' + os.environ['JOURNAL_USER']
                        + ':' + os.environ['JOURNAL_PASS']
                        + '@journal.sachachua.com/api/entries/' + zid)
    j = resp.json()
    if j and not re.search('^I thought about', j['Note']):
        return j['Note']


def get_color_map(filename, text=None):
    if text:
        together = filename + ' ' + text
    else:
        together = filename
    if re.search('r#(parenting|purple|life)', together):
        return {'9d9d9d': '8754a1', 'c9c9c9': 'e4c1d9'}  # parenting is purplish
    elif re.search(r'#(emacs|geek|tech|blue)', together):
        return {'9d9d9d': '2b64a9', 'c9c9c9': 'b3e3f1'}  # geeky stuff in light/dark blue
    else:
        return {'9d9d9d': '884636', 'c9c9c9': 'f6f396'}  # yellow highlighter, dark brown


def rename_set(old_name, new_name):
    if old_name != new_name:
        old_json = old_name[:-3] + 'json'
        new_json = new_name[:-3] + 'json'
        os.rename(old_name, new_name)
        os.rename(old_json, new_json)


def recolor_based_on_filename(filename):
    color_map = get_color_map(filename)
    recolored = recolor.map_colors(filename, color_map)
    # possibly rename based on the filename
    new_filename = re.sub(' #(purple|blue)', '', filename)
    rename_set(filename, new_filename)
    recolored.save(new_filename)


def move_processed_sketch(file):
    global refresh_counter
    if '#private' in file:
        output_dir = private_sketch_dir
    elif '#' in file:
        output_dir = public_sketch_dir
        refresh_counter = 3
    else:
        return file
    new_filename = os.path.join(output_dir, os.path.basename(file))
    rename_set(file, new_filename)
    return new_filename


def process_file(file):
    json_file = file[:-3] + 'json'
    # Check if a corresponding json file already exists
    if not os.path.exists(json_file):
        extract_text(client, file)
    if not re.search('[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2} ', file):
        file = maybe_rename(file)
    recolor_based_on_filename(file)
    move_processed_sketch(file)


def process_dir(folder_path):
    global processed_files
    # Iterate through all png files in the specified folder
    files = sorted(os.listdir(folder_path))
    for file in files:
        if file.endswith('.png') and '_' in file:
            print("Processing ", file)
            process_file(os.path.join(folder_path, file))


def daemon(folder_path, wait):
    global refresh_counter
    while True:
        process_dir(folder_path)
        time.sleep(wait)
        if refresh_counter > 0:
            refresh_counter = refresh_counter - 1
            if refresh_counter == 0:
                print("Reloading sketches")
                requests.get('https://' + os.environ['JOURNAL_USER'] + ':'
                             + os.environ['JOURNAL_PASS']
                             + '@sketches.sachachua.com/reload?python=1')


if __name__ == '__main__':
    # Create a set to store the names of processed files
    processed_files = set()
    if len(sys.argv) > 1:
        if os.path.isdir(sys.argv[1]):
            folder_path = sys.argv[1]
            daemon(folder_path, 300)
        else:
            for f in sys.argv[1:]:
                process_file(f)
    else:
        daemon(folder_path, 300)

It uses this script I wrote to recolor my sketches with Python.

I'm contemplating writing some annotation tools to make it easier to turn the detected text into useful text for searching or writing about because the sketches throw off the recognition (misrecognized text, low confidence) and the columns mess up the line wrapping. Low priority, though.

My handwriting (at least for numbers) is probably simple enough that I might be able to train Tesseract OCR to process that someday. And who knows, maybe some organization will release a pre-trained model for offline handwriting recognition that'll be as useful as OpenAI Whisper is for audio files. That would be neat!

Compiling selected blog posts into HTML and EPUB so I can annotate them

| blogging, 11ty, nodejs, supernote

[2023-01-04 Wed] Added a screenshot showing annotation.

I was thinking about how to prepare for my next 10-year review, since I'll turn 40 this year. I've been writing yearly reviews with some regularity and monthly reviews sporadically, and I figured it would be nice to have those posts in an EPUB so that I can read them on my e-reader and annotate them as I do my review.

I use the 11ty static site generator to publish my blog as HTML files, since I currently can't keep more than Emacs Lisp, Javascript, and Python in my brain. (No Hugo or Jekyll for me at the moment.) I briefly thought about getting 11ty to create that archive for me, but I realized it might be easier to just write it as an external script instead of trying to figure out how to get 11ty to export one thing conditionally.

One of the things I've configured 11ty to make is a JSON file that includes all of my posts with dates, titles, permalinks, and categories. It was easy to then parse this list and filter it to get the posts I wanted. I parsed the HTML out of the _site directory that 11ty produces instead of fetching the pages from my webserver. I got the images from my webserver, though, and I made a local cache and rewrote the URLs. That way, the EPUB conversion could include the images.

Download blog.js

blog.js
const blog = require('/home/sacha/proj/static-blog/_site/blog/all/index.json');
const cheerio = require('cheerio');
const base = '/home/sacha/proj/static-blog/_site';
const fs = require('fs');
const path = require('path');

function slugify(p) {
  return p.permalink.replace('/blog', 'post-').replace(/\//g, '-');
}

async function processPost(p) {
  console.log('Processing '+ p.permalink);
  let $ = cheerio.load(fs.readFileSync(base + p.permalink + 'index.html'));
  $('#comment').remove();
  let images = $('article img');
  await Promise.all(images.map((i, e) => {
    let url = $(e).attr('src');
    const outputFileName = 'images/' + path.basename(url).replace(/ |%20|%23/g, '-');
    $(e).attr('src', outputFileName);
    $(e).attr('style', 'max-height: 100%; max-width: 100%; ' + ($(e).attr('style') || ''));
    $(e).attr('srcset', null);
    $(e).attr('sizes', null);
    $(e).attr('width', null);
    $(e).attr('height', null);
    if (!fs.existsSync(outputFileName)) {
      console.log('fetch', outputFileName);
      return fetch(url).then(res => res.arrayBuffer()).then(data => {
        const buffer = Buffer.from(data);
        return fs.createWriteStream(outputFileName).write(buffer);
      });
    } else {
      console.log(outputFileName, 'exists');
      return null;
    }
  }));
  console.log('Done ' + p.permalink);
  let slug = slugify(p);
  $('article h2').attr('id', slug);
  let header = $('article header').html();
  let entry = $('article .entry').html();
  return `<article>${header}${entry}</article>`;
}

let last10 = blog.filter((p) => p.date >= '2013-08-01');
let posts = last10.filter((p) => p.categories.indexOf('yearly') >= 0)
    .concat(blog.filter((p) => p.title == 'Turning 30: A review of the last decade'))
    .concat(last10.filter((p) => p.categories.indexOf('monthly') >= 0));

let toc = '<h1>Table of Contents</h1><ul>' + posts.map((p) => {
  return `<li><a href="#${slugify(p)}">${p.title}</a></li>\n`;
}).join('') + '</ul>';

let content = posts.reduce(async (prev, val) => {
    return await prev + await processPost(val);
  }, '');
content.then((data) => {
  fs.writeFileSync('archive.html',
                   `<html><body>${toc}${data}</body></html>`);

});

This created an archive.html with my posts, using the images/ directory for the images. Then I used my shell script for converting and copying files to convert it to EPUB and copy it over.

On the SuperNote, I can highlight text by drawing square brackets around it. If I tap that text, I can write or draw underneath it. Here's what that looks like:

20230104_090739.png
Figure 1: Writing an annotation

These notes are collected into a "Digest" view, and I can export things from there. (Example: archive.pdf)

2023-01-04_09-23-57.png
Figure 2: Here's what that digest is like when exported.

(Hmm, maybe I should ask them about hiding the pencil icon…)

Anyway, I think that might be a good starting point for my review.

Building up my tech notes

| geek, supernote
  • [2023-01-04 Wed] Added extra CSS to force images to fit on the page
  • [2023-01-03 Tue] Updated shell script to use EPUB for more formats

A- wants me to sit with her at bedtime. She also wants to read a stack of books until she gets sleepy. This means I sometimes have an hour (or even two) of sitting quietly with her which I can use for writing, drawing, reading, or knitting, as long as I'm quiet. ("Mama, keepp your thoughts to yourself!")

My Supernote A5X supports EPUBs and PDFs, but doesn't support HTML files or my library's e-book platform (Libby), and I'm not too keen on the Kindle app. So I need to load it up with my own collection of books, manuals, API documentation, and notes.

Org Mode can export to EPUBs and PDFs well enough. If I make the output file a symbolic link to the same file in the Dropbox folder that's synchronized with my Supernote, I can re-export the EPUB and it will end up in the right place when I sync. I've started accumulating little snippets from the digest of my reading highlights, since putting them into Org Mode allows me to organize them and summarize them in different ways. It feels good to be collecting and organizing things I'm learning.

I plan to use this reading time to skim documentation for interesting things, since sometimes the challenges are more about knowing something exists and what it's called. Then I can copy the digests into my reference.org and export it as an EPUB or PDF, review that periodically, and maybe add some shortcuts to my Emacs configuration so that I can quickly jump to lines in my reference file.

HTML

The Supernote doesn't support HTML files, but I can convert HTML to PDFs with pandoc file.html -t latex -o file.pdf. This shell script copies files to my INBOX directory, converting HTML files along the way:

#!/bin/bash
INBOX=~/Dropbox/Supernote/INBOX
for FILE in "$@"; do
    if [[ "$FILE" == *.html ]]; then
        ebook-convert "$FILE" $INBOX/$(basename "$FILE" .html).epub --extra-css 'img { max-width: 100% !important; max-weight: 100% !important }'
        # or pdf: wkhtmltopdf --no-background "$FILE" $INBOX/$(basename "$FILE" .html).pdf
    elif [[ "$FILE" == *.xml ]]; then
        dbtoepub "$FILE" -o $INBOX/$(basename "$FILE" .xml).epub
    elif [[ "$FILE" == *.texi ]]; then
        texi2pdf "$FILE" -o $INBOX/$(basename "$FILE" .texi).pdf
    elif [[ "$FILE" == *.org ]]; then
        emacs -Q --batch "$FILE" --eval "(progn (package-initialize) (use-package 'ox-epub) (org-epub-export-to-epub))"
        cp "${FILE%.*}".epub $INBOX
    else
        cp "$FILE" $INBOX
    fi
done

Manpages

I'd like to be able to refer to manpages. I couldn't figure out how to get man -H to work with the Firefox inside a snap (it complained about having elevated permissions). I installed man2html and found the manpage for xdotool. zcat /usr/share/man/man1/xdotool.1.gz | man2html > /tmp/xdotool.html created the HTML file, and then I used ebook-convert /tmp/xdotool.html /tmp/xdotool.epub to create an EPUB file.

I tried getting the filename for the manpage by using the man command in Emacs, but I couldn't figure out how to get the filename from there. I remembered that Emacs has a woman command that displays manpages without using the external man command. That led me to woman-file-name, which gives me the path to the manpage given a command. Emacs handles uncompressing .gz files automatically, so everything's good to go from there.

(defvar my-supernote-inbox "~/Dropbox/Supernote/INBOX")
(defun my-save-manpage-to-supernote (path)
  (interactive (list (woman-file-name nil)))
  (let* ((base (file-name-base path))
         (temp-html (make-temp-file base nil ".html")))
    (with-temp-buffer
      (insert-file-contents path)
      (call-process-region (point-min) (point-max) "man2html" t t)
      (when (re-search-backward "Invalid Man Page" nil t)
        (delete-file temp-html)
        (error "Could not convert."))
      (write-file temp-html))
    (call-process "ebook-convert" nil (get-buffer-create "*temp*") nil temp-html
                  (expand-file-name (concat base ".epub") my-supernote-inbox))
    (delete-file temp-html)))

Info files

I turned the Elisp reference into a PDF by going to doc/lispref in my Emacs checkout and typing make elisp.pdf. It's 1470 pages long, so that should keep me busy for a while. Org Mode also has a make pdf target that uses texi2pdf to generate doc/org.pdf and doc/orgguide.pdf. Other .texi files could be converted with texi2pdf, or I can use makeinfo to create Docbook files and then use dbtoepub to convert them as in the shell script in the HTML section above.

Python documentation

I wanted to load the API documentation for autokey into one page for easy reference. The documentation at https://autokey.github.io/index.html was produced by epydoc, which doesn't support Python 3. I got to work using the sphinx-epytext extension. After I used sphinx-quickstart, I edited conf.py to include extensions = ['sphinx.ext.autodoc', 'sphinx_epytext', 'sphinx.ext.autosummary'], and I added the following to index.rst:

Welcome to autokey's documentation!
===================================

   .. autoclass:: autokey.scripting.Keyboard
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.Mouse
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.Store
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.QtDialog
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.System
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.QtClipboard
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.Window
      :members:
      :undoc-members:

   .. autoclass:: autokey.scripting.Engine
      :members:
      :undoc-members:

Then make pdf created a PDF. There's probably a way to get a proper table of contents, but it was a good start.

Linking to and exporting function definitions in Org Mode

| emacs

I'd like to write more blog posts about little Emacs hacks, and I'd like to do it with less effort. Including source code is handy even when it's missing some context from other functions defined in the same file, since sometimes people pick up ideas and having the source code right there means less flipping between links. When I'm working inside my config file or other literate programming documents, I can just write my blog post around the function definitions. When I'm talking about Emacs Lisp functions defined elsewhere, though, it's a little more annoying to copy the function definition and put it in a source block, especially if there are updates.

The following code creates a defun link type that exports the function definition. It works for functions that can be located with find-function, so only functions loaded from .el files, but that does what I need for now. Probably once I post this, someone will mention a much more elegant way to do things. Anyway, it makes it easier to use org-store-link to capture a link to the function, insert it into a blog post, navigate back to the function, and export HTML.

(defun my-org-defun-complete ()
  "Return function definitions."
  (completing-read
   "Function: "
   #'help--symbol-completion-table
   #'fboundp
   'confirm
   nil nil
   (and fn (symbol-name fn))))

(defun my-org-defun-export (symbol description format _)
  "Export the function."
  (save-window-excursion
    (find-function (intern symbol))
    (let ((function-body (buffer-substring (point)
                                           (progn (forward-sexp) (point)))))
      (pcase format
        ((or '11ty 'html)
         (format "<div class=\"org-src-container\">\n<details><summary>%s</summary><pre class=\"src src-emacs-lisp\">%s</pre></details></div>"
                 symbol
                 (org-html-do-format-code function-body "emacs-lisp" nil nil nil nil)))
        (`ascii function-body)
        (_ function-body)))))

(defun my-org-defun-store ()
  "Store a link to the function."
  (when (derived-mode-p 'emacs-lisp-mode)
    (org-link-store-props :type "defun"
                          :link (concat "defun:" (lisp-current-defun-name)))))

(defun my-org-defun-open (symbol _)
  "Jump to the function definition."
  (find-function (intern symbol)))

(org-link-set-parameters "defun" :follow #'my-org-defun-open
                         :export #'my-org-defun-export
                         :complete #'my-org-defun-complete
                         :store #'my-org-defun-store)

For example, if I have something like the following Org markup:

defun:emacsconf-prep-agenda

I can pull in the definition of emacsconf-prep-agenda from emacsconf.el, which you can find in the emacsconf-el repository.

emacsconf-prep-agenda
(defun emacsconf-prep-agenda ()
  (interactive)
  (let* ((org-agenda-custom-commands
         `(("a" "Agenda"
            ((tags-todo "-PRIORITY=\"C\"-SCHEDULED={.}-nextyear"
                        ((org-agenda-files (list ,emacsconf-notebook))
                         (org-agenda-sorting-strategy '(priority-down effort-up))))
             (agenda ""
                     ((org-agenda-files (list ,emacsconf-notebook))
                      (org-agenda-span 7)))
             )))))
    (org-agenda nil "a")))

This is part of my Emacs configuration.

EmacsConf backstage: Using TRAMP and timers to run two tracks semi-automatically

| emacs, emacsconf, org

In previous years, organizers streamed the video feeds for EmacsConf from their own computers to the Icecast server, which was a little challenging because of CPU load. A server shared by a volunteer had a 6-core Intel Xeon E5-2420 with 48 GB of RAM, which turned out to be enough horsepower to run OBS for both the general and development track for EmacsConf 2022. One of the advantages of this setup was that I could write some Emacs Lisp to automatically play recorded intros and talk videos at scheduled times, right from the large Org file that had all the conference details. I used SCHEDULED: properties to indicate when talks should play, and that was picked up by another function that took the Org entry properties and put them into a plist.

This function scheduled the timers:

emacsconf-stream-schedule-timers
(defun emacsconf-stream-schedule-timers (&optional info)
  "Schedule PLAYING for the rest of talks and CLOSED_Q for recorded talks."
  (interactive)
  (emacsconf-stream-cancel-all-timers)
  (setq info (emacsconf-prepare-for-display (emacsconf-filter-talks (or info (emacsconf-get-talk-info)))))
  (let ((now (current-time)))
    (mapc (lambda (talk)
            (when (and (time-less-p now (plist-get talk :start-time)))
              (emacsconf-stream-schedule-talk-status-change talk (plist-get talk :start-time) "PLAYING"
                                                            `(:title (concat "Starting " (plist-get talk :slug)))))
            (when (and
                   (plist-get talk :video-file)
                   (plist-get talk :qa-time)
                   (not (string-match "none" (or (plist-get talk :q-and-a) "none")))
                   (null (plist-get talk :stream-files)) ;; can't tell when this is
                   (time-less-p now (plist-get talk :qa-time)))
              (emacsconf-stream-schedule-talk-status-change talk (plist-get talk :qa-time) "CLOSED_Q"
                                                            `(:title (concat "Q&A for " (plist-get talk :slug) " (" (plist-get talk :q-and-a) ")"))))
            )
          info)))

It turns out that TRAMP doesn't like being called from timers if there's a chance that two TRAMP processes might run at the same time. I got "Forbidden reentrant call of Tramp" errors when that happened. There was an easy fix, though. I adjusted the schedules of the talks so that they started at least a minute apart.

Sometimes I wanted to cancel just one timer:

emacsconf-stream-cancel-timer
(defun emacsconf-stream-cancel-timer (id)
  "Cancel a timer by ID."
  (interactive (list
                (completing-read
                 "ID: "
                 (lambda (string pred action)
                    (if (eq action 'metadata)
                        `(metadata (display-sort-function . ,#'identity))
                      (complete-with-action action
                                            (sort
                                             (seq-filter (lambda (o)
                                                           (and (timerp (cdr o))
                                                                (not (timer--triggered (cdr o)))))
                                                         emacsconf-stream-timers)
                                             (lambda (a b) (string< (car a) (car b))))
                                            string pred))))))
  (when (timerp (assoc-default id emacsconf-stream-timers))
    (cancel-timer (assoc-default id emacsconf-stream-timers))
    (setq emacsconf-stream-timers
          (delq (assoc id emacsconf-stream-timers)
                (seq-filter (lambda (o)
                              (and (timerp (cdr o))
                                   (not (timer--triggered (cdr o)))))
                            emacsconf-stream-timers)))))

and schedule just one timer manually:

emacsconf-stream-schedule-talk-status-change
(defun emacsconf-stream-schedule-talk-status-change (talk time new-status &optional notification)
  "Schedule a one-off timer for TALK at TIME to set it to NEW-STATUS."
  (interactive (list (emacsconf-complete-talk-info)
                     (read-string "Time: ")
                     (completing-read "Status: " (mapcar 'car emacsconf-status-types))))
  (require 'diary-lib)
  (setq talk (emacsconf-resolve-talk talk))
  (let* ((converted
          (cond
           ((listp time) time)
           ((timer-duration time) (timer-relative-time nil (timer-duration time)))
           (t                           ; HH:MM
            (date-to-time (concat (format-time-string "%Y-%m-%d" nil emacsconf-timezone)
                                  "T"
                                  (string-pad time 5 ?0 t) 
                                  emacsconf-timezone-offset)))))
         (timer-id (concat (format-time-string "%m-%dT%H:%M" converted)
                           "-"
                           (plist-get talk :slug)
                           "-"
                           new-status)))
    (emacsconf-stream-cancel-timer timer-id) 
    (add-to-list 'emacsconf-stream-timers
                  (cons
                   timer-id
                   (run-at-time time converted #'emacsconf-stream-update-talk-status-from-timer
                                talk new-status
                                notification)))))

The actual playing of talks happened using functions that were called from org-after-todo-state-change-hook. I wrote a function that extracted the talk information and then called my own list of functions.

emacsconf-org-after-todo-state-change
(defun emacsconf-org-after-todo-state-change ()
  "Run all the hooks in `emacsconf-todo-hooks'.
If an `emacsconf-todo-hooks' entry is a list, run it only for the
tracks with the ID in the cdr of that list."
  (let* ((talk (emacsconf-get-talk-info-for-subtree))
         (track (emacsconf-get-track (plist-get talk :track))))
    (mapc
     (lambda (hook-entry)
       (cond
        ((symbolp hook-entry) (funcall hook-entry talk))
        ((member (plist-get track :id) (cdr hook-entry))
         (funcall (car hook-entry) talk))))
     emacsconf-todo-hooks)))

For example, this function played the recorded intro and the talk:

emacsconf-stream-play-talk-on-change
(defun emacsconf-stream-play-talk-on-change (talk)
  "Play the talk."
  (interactive (list (emacsconf-complete-talk-info)))
  (setq talk (emacsconf-resolve-talk talk))
  (when (or (not (boundp 'org-state)) (string= org-state "PLAYING"))
    (if (plist-get talk :stream-files)
        (progn
          (emacsconf-stream-track-ssh
           talk
           "overlay"
           (plist-get talk :slug))
          (emacsconf-stream-track-ssh
           talk
           (append
            (list
             "nohup"
             "mpv")
            (split-string-and-unquote (plist-get talk :stream-files))
            (list "&"))))
      (emacsconf-stream-track-ssh
       talk
       (cons
        "nohup"
        (cond
         ((and
           (plist-get talk :recorded-intro)
           (plist-get talk :video-file)) ;; recorded intro and recorded talk
          (message "should automatically play intro and recording")
          (list "play-with-intro" (plist-get talk :slug))) ;; todo deal with stream files
         ((and
           (plist-get talk :recorded-intro)
           (null (plist-get talk :video-file))) ;; recorded intro and live talk; play the intro and join BBB
          (message "should automatically play intro; join %s" (plist-get talk :bbb-backstage))
          (list "intro" (plist-get talk :slug)))
         ((and
           (null (plist-get talk :recorded-intro))
           (plist-get talk :video-file)) ;; live intro and recorded talk, show slide and use Mumble; manually play talk
          (message "should show intro slide; play %s afterwards" (plist-get talk :slug))
          (list "intro" (plist-get talk :slug)))
         ((and
           (null (plist-get talk :recorded-intro))
           (null (plist-get talk :video-file))) ;; live intro and live talk, join the BBB
          (message "join %s for live intro and talk" (plist-get talk :bbb-backstage))
          (list "bbb" (plist-get talk :slug)))))))))

and this function handled IRC announcements when the talk state changed:

emacsconf-erc-announce-on-change
(defun emacsconf-erc-announce-on-change (talk)
  "Announce talk."
  (let ((func
         (pcase org-state
           ("PLAYING" #'erc-cmd-NOWPLAYING)
           ("CLOSED_Q" #'erc-cmd-NOWCLOSEDQ)
           ("OPEN_Q" #'erc-cmd-NOWOPENQ)
           ("UNSTREAMED_Q" #'erc-cmd-NOWUNSTREAMEDQ)
           ("TO_ARCHIVE" #'erc-cmd-NOWDONE))))
    (when func
      (funcall func talk))))

The actual announcements were handled by something like this:

erc-cmd-NOWCLOSEDQ
(defun erc-cmd-NOWCLOSEDQ (talk)
  "Announce TALK has started Q&A, but the host has not yet opened it up."
  (interactive (list (emacsconf-complete-talk-info)))
  (when (stringp talk) (setq talk (or (emacsconf-find-talk-info talk) (error "Could not find talk %s" talk))))
  (if (emacsconf-erc-recently-announced (format "-- Q&A beginning for \"%s\"" (plist-get talk :slug)))
      (message "Recently announced, skipping")
    (emacsconf-erc-with-channels (list (concat "#" (plist-get talk :channel)))
      (erc-send-message (format "-- Q&A beginning for \"%s\" (%s) Watch: %s Add notes/questions: %s"
                                (plist-get talk :title)
                                (plist-get talk :qa-info)
                                (plist-get talk :watch-url)
                                (plist-get talk :pad-url))))  
    (emacsconf-erc-with-channels (list emacsconf-erc-hallway emacsconf-erc-org)
      (erc-send-message (format "-- Q&A beginning for \"%s\" in the %s track (%s) Watch: %s Add notes/questions: %s . Chat: #%s"
                                (plist-get talk :title)
                                (plist-get talk :track)
                                (plist-get talk :qa-info)
                                (plist-get talk :watch-url)
                                (plist-get talk :pad-url)
                                (plist-get talk :channel))))))

All that code meant that during the actual conference, my role was mostly just worrying, and occasionally starting up the Q&A (if I wasn't sure if the code would do it right). The shell scripts I wrote made it easy for the other organizers to take over the second part as they saw how it worked.

Yay timers, Emacs, and TRAMP!

You can find the latest versions of these functions in the emacsconf-el repository.

2023-01-02 Emacs news

| emacs, emacs-news

Links from reddit.com/r/emacs, r/orgmode, r/spacemacs, r/planetemacs, Hacker News, lobste.rs, planet.emacslife.com, YouTube, the Emacs NEWS file, Emacs Calendar, emacs-devel, and lemmy/c/emacs. Thanks to Andrés Ramírez for emacs-devel links. Do you have an Emacs-related link or announcement? Please e-mail me at sacha@sachachua.com. Thank you!