Initial commit.

ocado_grocy/postprocess.py

@@ -0,0 +1,112 @@
+"""Enrich products belonging to the orders just processed, including reruns."""
+import hashlib
+import os
+from pathlib import Path
+
+import click
+
+from .grocy import Grocy, imported_products
+from .history import private_json
+from .images import run_images
+from .openfoodfacts import run_openfoodfacts
+from .enrichment_retry import RetryQueue
+
+
+def enrich_orders(config, config_path, archive, order_ids, *, images=True, openfoodfacts=True):
+    if not order_ids or not (images or openfoodfacts):
+        return {'errors':[]}
+    connection = config.get('grocy', {})
+    client = Grocy(os.environ.get('GROCY_URL', connection.get('url', '')),
+                   os.environ.get('GROCY_API_KEY', connection.get('api_key', '')))
+    state = Path(config.get('import', {}).get('state_file', 'imports.sqlite3'))
+    if not state.is_absolute():
+        state = config_path.resolve().parent / state
+    products = imported_products(client, state, order_ids)
+    key = order_ids[0] if len(order_ids) == 1 and order_ids[0].isdigit() else hashlib.sha256(','.join(sorted(order_ids)).encode()).hexdigest()[:16]
+    output = archive / 'enrichment' / key
+    queue = RetryQueue(state, client.url)
+    summary = {'server':client.url, 'order_ids':list(order_ids), 'product_ids':[p['id'] for p in products], 'errors':[]}
+    if not products:
+        queue.close()
+        private_json(output / 'summary.json', summary)
+        return summary
+    if images:
+        try:
+            click.echo(f'Adding missing pictures for {len(products)} imported products...')
+            queue.start('images', products)
+            result = run_images(client, products, output / 'images.json', raise_errors=False)
+            queue.results('images', result['products'])
+            summary['images'] = result['counts']
+            if result['counts'].get('error'):
+                summary['errors'].append(f"Images: {result['counts']['error']} products failed")
+        except Exception as exc:
+            summary['errors'].append(f'Images: {exc}')
+    if openfoodfacts:
+        try:
+            queue.start('openfoodfacts', products)
+            contact = config.get('openfoodfacts', {}).get('contact')
+            if not contact:
+                raise ValueError('Set [openfoodfacts].contact in config.toml or use --no-openfoodfacts')
+            click.echo(f'Matching {len(products)} imported products on Open Food Facts...')
+            result = run_openfoodfacts(client, products, archive / 'openfoodfacts', contact)
+            queue.results('openfoodfacts', result['products'])
+            private_json(output / 'openfoodfacts.json', result)
+            summary['openfoodfacts'] = result['counts']
+            if result['counts'].get('error'):
+                summary['errors'].append(f"Open Food Facts: {result['counts']['error']} products failed; see {output / 'openfoodfacts.json'}")
+        except Exception as exc:
+            summary['errors'].append(f'Open Food Facts: {exc}')
+    queue.close()
+    private_json(output / 'summary.json', summary)
+    return summary
+
+
+def retry_enrichment(config, config_path, archive, *, images=True, openfoodfacts=True):
+    connection = config.get('grocy', {})
+    client = Grocy(os.environ.get('GROCY_URL', connection.get('url','')),
+                   os.environ.get('GROCY_API_KEY', connection.get('api_key','')))
+    state = config_path.resolve().parent / config.get('import',{}).get('state_file','imports.sqlite3')
+    if not state.exists():
+        return {'errors':[]}
+    queue = RetryQueue(state, client.url)
+    try:
+        # Journal scope protects other Grocy installations and non-imported products.
+        tables = {r[0] for r in queue.db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
+        if 'imports' not in tables:
+            return {'errors':[]}
+        allowed = {int(r[0]) for r in queue.db.execute("SELECT DISTINCT product_id FROM imports WHERE server=? AND status='done'",(client.url,))}
+        queue.migrate_reports(archive, allowed)
+        enabled = [('images',images),('openfoodfacts',openfoodfacts)]
+        summary = {'errors':[]}
+        live = None
+        for stage, active in enabled:
+            pending = queue.pending(stage)
+            if not active or not pending:
+                continue
+            if live is None:
+                live = {int(p['id']):p for p in client.get('/objects/products')}
+            products = [live[pid] for pid in sorted(pending & allowed & live.keys())]
+            queue.results(stage,[{'product_id':pid,'status':'deleted_or_out_of_scope'} for pid in pending - {int(p['id']) for p in products}])
+            if not products:
+                continue
+            click.echo(f'Retrying {stage} for {len(products)} previously failed products...')
+            output = archive / 'enrichment' / 'retries'
+            try:
+                if stage == 'images':
+                    result = run_images(client, products, output/'images.json', raise_errors=False)
+                else:
+                    contact = config.get('openfoodfacts',{}).get('contact')
+                    if not contact:
+                        raise ValueError('Set [openfoodfacts].contact in config.toml')
+                    result = run_openfoodfacts(client, products, archive/'openfoodfacts',contact)
+                    private_json(output/'openfoodfacts.json',result)
+                queue.results(stage,result['products'])
+                summary[stage] = result['counts']
+                if result['counts'].get('error'):
+                    summary['errors'].append(f"{stage}: {result['counts']['error']} retries failed")
+            except Exception as exc:
+                summary['errors'].append(f'{stage}: {exc}')
+        private_json(archive/'enrichment'/'retry-summary.json',summary)
+        return summary
+    finally:
+        queue.close()