crontab

# Edit this file to introduce tasks to be run by cron.
#
# Each task to run has to be defined through a single line
# indicating with different fields when the task will be run
# and what command to run for the task
#
# To define the time you can provide concrete values for
# minute (m), hour (h), day of month (dom), month (mon),
# and day of week (dow) or use '*' in these fields (for 'any').#
# Notice that tasks will be started based on the cron's system
# daemon's notion of time and timezones.
#
# Output of the crontab jobs (including errors) is sent through
# email to the user the crontab file belongs to (unless redirected).
#
# For example, you can run a backup of all your user accounts
# at 5 a.m every week with:
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
#
# For more information see the manual pages of crontab(5) and cron(8)
#
# m h  dom mon dow   command

# TODO(david): Don't redirect stderr and instead mail stderr to us
# TODO(david): Set up logrotate on generated log files

SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games

# Make Python use UTF-8 encoding when the encoding of the output device is not
# known. Otherwise, Python will throw a UnicodeEncodeError when a unicode
# character output is piped. From http://stackoverflow.com/a/4027726
PYTHONIOENCODING=utf-8

# Make log output sequential and actually comprehensible.
PYTHONUNBUFFERED=true

# Restart Rethinkdb and the server.
# TODO(david): MAJOR BANDAID HACK!! This is here because Rethinkdb has been
#     dying every few days for some reason. I have tried updating RethinkDB's patch
#     version, but the problem still seems to be there. There are no errors
#     output to RethinkDB's log file when this happens. This may or may not
#     have to do with Rethinkdb gobbling up too much memory. Experiment with
#     changing Rethinkdb cache sizes (though it looks like Rethinkdb is already
#     using the minimum possible memory), or just ask on #rethinkdb.
42 5 * * * $HOME/vim-awesome/tools/restart_rethinkdb_and_server.sh

# Data pipeline to update all of our plugins daily

# 1. Scrape from all vim.org plugins
13 6 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/scrape.py -s vim.org 8000 >> $HOME/logs/cron/vimorg.log 2>&1

# 2. Scrape from all github.com/vim-scripts plugins. This scrapes about 100 plugins per GitHub API request.
13 7 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/scrape.py -s github-vim-scripts 8000 >> $HOME/logs/cron/github-vim-scripts.log 2>&1

# 3. Scrape all dotfiles pushed since the last time we scraped them. This uses about 2-3 GitHub API requests per dotfile repo scraped.
13 8 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/scrape.py -s github-dotfiles 1600 >> $HOME/logs/cron/github-dotfiles.log 2>&1

# 5. Update our index of known GitHub plugin repos, as well as the count of dotfiles repos that reference each plugin repo.
13 9 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/build_github_index.py >> $HOME/logs/cron/build-github-index.log 2>&1

# 6. Scrape GitHub plugins. This uses about 4 GitHub API requests per plugin repo scraped.
13 10 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/scrape.py -s github-plugins 1200 >> $HOME/logs/cron/github-plugins.log 2>&1
13 11 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/scrape.py -s github-plugins 1200 >> $HOME/logs/cron/github-plugins.log 2>&1
13 12 * * * PYTHONPATH=$HOME/vim-awesome python $HOME/vim-awesome/tools/scrape/scrape.py -s github-plugins 1200 >> $HOME/logs/cron/github-plugins.log 2>&1

# Daily DB dumps which we can back up to Tarsnap. Logrotate will delete old dumps.
13 13 * * * ( cd $HOME/dumps && rethinkdb dump )