112 lines
5.7 KiB
Python
112 lines
5.7 KiB
Python
"""Enrich products belonging to the orders just processed, including reruns."""
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import click
|
|
|
|
from .grocy import Grocy, imported_products
|
|
from .history import private_json
|
|
from .images import run_images
|
|
from .openfoodfacts import run_openfoodfacts
|
|
from .enrichment_retry import RetryQueue
|
|
|
|
|
|
def enrich_orders(config, config_path, archive, order_ids, *, images=True, openfoodfacts=True):
|
|
if not order_ids or not (images or openfoodfacts):
|
|
return {'errors':[]}
|
|
connection = config.get('grocy', {})
|
|
client = Grocy(os.environ.get('GROCY_URL', connection.get('url', '')),
|
|
os.environ.get('GROCY_API_KEY', connection.get('api_key', '')))
|
|
state = Path(config.get('import', {}).get('state_file', 'imports.sqlite3'))
|
|
if not state.is_absolute():
|
|
state = config_path.resolve().parent / state
|
|
products = imported_products(client, state, order_ids)
|
|
key = order_ids[0] if len(order_ids) == 1 and order_ids[0].isdigit() else hashlib.sha256(','.join(sorted(order_ids)).encode()).hexdigest()[:16]
|
|
output = archive / 'enrichment' / key
|
|
queue = RetryQueue(state, client.url)
|
|
summary = {'server':client.url, 'order_ids':list(order_ids), 'product_ids':[p['id'] for p in products], 'errors':[]}
|
|
if not products:
|
|
queue.close()
|
|
private_json(output / 'summary.json', summary)
|
|
return summary
|
|
if images:
|
|
try:
|
|
click.echo(f'Adding missing pictures for {len(products)} imported products...')
|
|
queue.start('images', products)
|
|
result = run_images(client, products, output / 'images.json', raise_errors=False)
|
|
queue.results('images', result['products'])
|
|
summary['images'] = result['counts']
|
|
if result['counts'].get('error'):
|
|
summary['errors'].append(f"Images: {result['counts']['error']} products failed")
|
|
except Exception as exc:
|
|
summary['errors'].append(f'Images: {exc}')
|
|
if openfoodfacts:
|
|
try:
|
|
queue.start('openfoodfacts', products)
|
|
contact = config.get('openfoodfacts', {}).get('contact')
|
|
if not contact:
|
|
raise ValueError('Set [openfoodfacts].contact in config.toml or use --no-openfoodfacts')
|
|
click.echo(f'Matching {len(products)} imported products on Open Food Facts...')
|
|
result = run_openfoodfacts(client, products, archive / 'openfoodfacts', contact)
|
|
queue.results('openfoodfacts', result['products'])
|
|
private_json(output / 'openfoodfacts.json', result)
|
|
summary['openfoodfacts'] = result['counts']
|
|
if result['counts'].get('error'):
|
|
summary['errors'].append(f"Open Food Facts: {result['counts']['error']} products failed; see {output / 'openfoodfacts.json'}")
|
|
except Exception as exc:
|
|
summary['errors'].append(f'Open Food Facts: {exc}')
|
|
queue.close()
|
|
private_json(output / 'summary.json', summary)
|
|
return summary
|
|
|
|
|
|
def retry_enrichment(config, config_path, archive, *, images=True, openfoodfacts=True):
|
|
connection = config.get('grocy', {})
|
|
client = Grocy(os.environ.get('GROCY_URL', connection.get('url','')),
|
|
os.environ.get('GROCY_API_KEY', connection.get('api_key','')))
|
|
state = config_path.resolve().parent / config.get('import',{}).get('state_file','imports.sqlite3')
|
|
if not state.exists():
|
|
return {'errors':[]}
|
|
queue = RetryQueue(state, client.url)
|
|
try:
|
|
# Journal scope protects other Grocy installations and non-imported products.
|
|
tables = {r[0] for r in queue.db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
if 'imports' not in tables:
|
|
return {'errors':[]}
|
|
allowed = {int(r[0]) for r in queue.db.execute("SELECT DISTINCT product_id FROM imports WHERE server=? AND status='done'",(client.url,))}
|
|
queue.migrate_reports(archive, allowed)
|
|
enabled = [('images',images),('openfoodfacts',openfoodfacts)]
|
|
summary = {'errors':[]}
|
|
live = None
|
|
for stage, active in enabled:
|
|
pending = queue.pending(stage)
|
|
if not active or not pending:
|
|
continue
|
|
if live is None:
|
|
live = {int(p['id']):p for p in client.get('/objects/products')}
|
|
products = [live[pid] for pid in sorted(pending & allowed & live.keys())]
|
|
queue.results(stage,[{'product_id':pid,'status':'deleted_or_out_of_scope'} for pid in pending - {int(p['id']) for p in products}])
|
|
if not products:
|
|
continue
|
|
click.echo(f'Retrying {stage} for {len(products)} previously failed products...')
|
|
output = archive / 'enrichment' / 'retries'
|
|
try:
|
|
if stage == 'images':
|
|
result = run_images(client, products, output/'images.json', raise_errors=False)
|
|
else:
|
|
contact = config.get('openfoodfacts',{}).get('contact')
|
|
if not contact:
|
|
raise ValueError('Set [openfoodfacts].contact in config.toml')
|
|
result = run_openfoodfacts(client, products, archive/'openfoodfacts',contact)
|
|
private_json(output/'openfoodfacts.json',result)
|
|
queue.results(stage,result['products'])
|
|
summary[stage] = result['counts']
|
|
if result['counts'].get('error'):
|
|
summary['errors'].append(f"{stage}: {result['counts']['error']} retries failed")
|
|
except Exception as exc:
|
|
summary['errors'].append(f'{stage}: {exc}')
|
|
private_json(archive/'enrichment'/'retry-summary.json',summary)
|
|
return summary
|
|
finally:
|
|
queue.close()
|