ArchiveBox/archivebox/core/migrations/0007_archiveresult.py

# Generated by Django 3.0.8 on 2020-11-04 12:25

import json
from pathlib import Path

from django.db import migrations, models
import django.db.models.deletion

from config import CONFIG


def forwards_func(apps, schema_editor):
    from core.models import EXTRACTORS

    Snapshot = apps.get_model("core", "Snapshot")
    ArchiveResult = apps.get_model("core", "ArchiveResult")

    snapshots = Snapshot.objects.all()
    for snapshot in snapshots:
        out_dir = Path(CONFIG['ARCHIVE_DIR']) / snapshot.timestamp

        try:
            with open(out_dir / "index.json", "r") as f:
                fs_index = json.load(f)
        except Exception as e:
            continue

        history = fs_index["history"]

        for extractor in history:
            for result in history[extractor]:
                ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=json.dumps(result["cmd"]), cmd_version=result["cmd_version"], 
                start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"])



def reverse_func(apps, schema_editor):
    ArchiveResult = apps.get_model("core", "ArchiveResult")
    ArchiveResult.objects.all().delete()


class Migration(migrations.Migration):

    dependencies = [
        ('core', '0006_auto_20201012_1520'),
    ]

    operations = [
        migrations.CreateModel(
            name='ArchiveResult',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('cmd', models.CharField(default='', max_length=500)),
                ('pwd', models.CharField(default='', max_length=200)),
                ('cmd_version', models.CharField(default='', max_length=20)),
                ('status', models.CharField(max_length=10)),
                ('output', models.CharField(default='', max_length=500)),
                ('start_ts', models.DateTimeField()),
                ('end_ts', models.DateTimeField()),
                ('extractor', models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('wget', 'wget'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('headers', 'headers'), ('archive_org', 'archive_org')], max_length=20)),
                ('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Snapshot')),
            ],
        ),
        migrations.RunPython(forwards_func, reverse_func),
    ]