Added things we needed for the search worker service.

sbenv.py

@@ -26,3 +26,15 @@ def get_admin_key():
     if ak is not None:
         ak = ak.strip()
     return ak
+
+def get_worker_datadir():
+    wdd = os.getenv('SB_WORKER_DATADIR')
+    if wdd is not None:
+        wdd = wdd.strip()
+    return wdd
+
+def get_submit_url():
+    su = os.getenv('SB_SUBMIT_URL')
+    if su is not None:
+        su = su.strip()
+    return su

search-wrapper.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export SB_ADMIN_KEY=
+export SB_WORKER_DATADIR=
+export SB_SUBMIT_URL=
+
+exec $(dirname $(realpath $0))/searchwkr.py

searchwkr.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import json
+from datetime import datetime
+
+import inventory
+import searchlib
+import sbenv
+
+def main(argv):
+    # Setup
+    searchlib.prep_nltk()
+
+    # Actually do the query.
+    today = datetime.now()
+    recents_before = load_recents()
+    res = searchlib.query_day(today, preloadurls=recents_before)
+    print(json.dumps(res, indent='  '))
+
+    # Save the urls we've seen now.
+    recents_before['urls'].append(list(map(lambda e: e['u'], res)))
+    save_recents(recents_before)
+
+    # Now send them up.
+    ok = True
+    for e in res:
+        print('uploading', e['u'], '...')
+        try:
+            send_item(today, e)
+        except Exception as e:
+            print('failed!', str(e))
+            ok = False
+
+    return 0 if ok else 1
+
+def send_item(day, ent):
+    body = {
+        'date': day.strftime(inventory.DATE_FORMAT),
+        'desc': ent
+    }
+
+    h = {
+        'Authorization': 'Bearer %s' % sbenv.get_admin_key(),
+    }
+
+    surl = sbenv.get_submit_url()
+    r = requests.post(surl, headers=h, data=json.dumps(body))
+    r.raise_for_status()
+
+def get_recents_path():
+    return os.path.join(sbenv.get_worker_datadir(), 'recents.json')
+
+# Loads the recents file, creating a new one if it's from a different day.
+def load_recents():
+    cdate = datetime.now().strftime(inventory.DATE_FORMAT)
+    rp = get_recents_path()
+
+    if not os.path.exists(rp):
+        return {
+            'cdate': cdate,
+            'urls': []
+        }
+
+    with open(rp, 'r') as f:
+        j = json.load(f)
+        if j['cdate'] != cdate:
+            return {
+                'cdate': cdate,
+                'urls': []
+            }
+        else:
+            return j
+
+# Saves the recents file.
+def save_recents(recents):
+    with open(get_recents_path(), 'w') as f:
+        json.dump(recents, f, indent='  ')
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

storrowed-search.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=storrowed.boston search worker
+Wants=storrowed-trigger.timer
+
+[Service]
+Type=oneshot
+WorkingDirectory=/opt/storrowed.boston
+ExecStart=/usr/bin/pipenv run /opt/storrowed.boston/search-wrapper.sh
+
+[Install]
+WantedBy=multi-user.target

storrowed-trigger.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=storrowed.boston search trigger
+Requires=storrowed-search.service
+
+[Timer]
+Unit=mastoreddit.service
+OnCalendar=*-*-* 2,8,14,20:00:00
+
+[Install]
+WantedBy=timers.target