add py-machineid lib for new machine app

archivebox/core/admin.py

@@ -29,6 +29,7 @@ from core.mixins import SearchResultsAdminMixin
 from api.models import APIToken
 from abid_utils.admin import ABIDModelAdmin
 from queues.tasks import bg_archive_links, bg_add
+from machine.models import Machine, NetworkInterface
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
@@ -778,3 +779,53 @@ class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
     list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
     sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
     readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
+
+
+@admin.register(Machine, site=archivebox_admin)
+class MachineAdmin(ABIDModelAdmin):
+    list_display = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
+    sort_fields = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
+    # search_fields = ('id', 'abid', 'guid', 'hostname', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release')
+    
+    readonly_fields = ('guid', 'created_at', 'modified_at', 'abid_info', 'ips')
+    fields = (*readonly_fields, 'hostname', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release', 'stats')
+
+    list_filter = ('hw_in_docker', 'hw_in_vm', 'os_arch', 'os_family', 'os_platform')
+    ordering = ['-created_at']
+    list_per_page = 100
+
+    @admin.display(
+        description='Public IP',
+        ordering='networkinterface__ip_public',
+    )
+    def ips(self, machine):
+        return format_html(
+            '<a href="/admin/machine/networkinterface/?q={}"><b><code>{}</code></b></a>',
+            machine.abid,
+            ', '.join(machine.networkinterface_set.values_list('ip_public', flat=True)),
+        )
+
+@admin.register(NetworkInterface, site=archivebox_admin)
+class NetworkInterfaceAdmin(ABIDModelAdmin):
+    list_display = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
+    sort_fields = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
+    search_fields = ('abid', 'machine__abid', 'iface', 'ip_public', 'ip_local', 'mac_address', 'dns_server', 'hostname', 'isp', 'city', 'region', 'country')
+    
+    readonly_fields = ('machine', 'created_at', 'modified_at', 'abid_info', 'mac_address', 'ip_public', 'ip_local', 'dns_server')
+    fields = (*readonly_fields, 'iface', 'hostname', 'isp', 'city', 'region', 'country')
+
+    list_filter = ('isp', 'country', 'region')
+    ordering = ['-created_at']
+    list_per_page = 100
+
+    @admin.display(
+        description='Machine',
+        ordering='machine__abid',
+    )
+    def machine_info(self, iface):
+        return format_html(
+            '<a href="/admin/machine/machine/{}/change"><b><code>[{}]</code></b> &nbsp; {}</a>',
+            iface.machine.id,
+            iface.machine.abid,
+            iface.machine.hostname,
+        )

archivebox/core/settings.py

@@ -98,7 +98,8 @@ INSTALLED_APPS = [
     'django_object_actions',     # provides easy Django Admin action buttons on change views       https://github.com/crccheck/django-object-actions
 
     # Our ArchiveBox-provided apps
-    #'config',                    # ArchiveBox config settings
+    #'config',                   # ArchiveBox config settings (loaded as a plugin, don't need to add it here)
+    'machine',                   # handles collecting and storing information about the host machine, network interfaces, installed binaries, etc.
     'queues',                    # handles starting and managing background workers and processes
     'abid_utils',                # handles ABID ID creation, handling, and models
     'core',                      # core django model with Snapshot, ArchiveResult, etc.

archivebox/machine/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class MachineConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    
+    name = 'machine'
+    verbose_name = 'Machine Info'

archivebox/machine/detect.py

@@ -0,0 +1,317 @@
+import os
+import json
+import socket
+import urllib.request
+from typing import Dict, Any
+from pathlib import Path
+import subprocess
+import platform
+import tempfile
+from datetime import datetime
+
+import psutil
+import machineid             # https://github.com/keygen-sh/py-machineid
+
+from rich import print
+
+PACKAGE_DIR = Path(__file__).parent
+DATA_DIR = Path('.').resolve()
+
+def get_vm_info():
+    hw_in_docker = bool(os.getenv('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE'))
+    hw_in_vm = False
+    try:
+        # check for traces of docker/containerd/podman in cgroup
+        with open('/proc/self/cgroup', 'r') as procfile:
+            for line in procfile:
+                cgroup = line.strip()  # .split('/', 1)[-1].lower()
+                if 'docker' in cgroup or 'containerd' in cgroup or 'podman' in cgroup:
+                    hw_in_docker = True
+    except Exception:
+        pass
+    
+    hw_manufacturer = 'Docker' if hw_in_docker else 'Unknown'
+    hw_product = 'Container' if hw_in_docker else 'Unknown'
+    hw_uuid = machineid.id()
+    
+    if platform.system().lower() == 'darwin':
+        # Get macOS machine info
+        hw_manufacturer = 'Apple'
+        hw_product = 'Mac'
+        try:
+            # Hardware:
+            #     Hardware Overview:
+            #       Model Name: Mac Studio
+            #       Model Identifier: Mac13,1
+            #       Model Number: MJMV3LL/A
+            #       ...
+            #       Serial Number (system): M230YYTD77
+            #       Hardware UUID: 39A12B50-1972-5910-8BEE-235AD20C8EE3
+            #       ...
+            result = subprocess.run(['system_profiler', 'SPHardwareDataType'], capture_output=True, text=True, check=True)
+            for line in result.stdout.split('\n'):
+                if 'Model Name:' in line:
+                    hw_product = line.split(':', 1)[-1].strip()
+                elif 'Model Identifier:' in line:
+                    hw_product += ' ' + line.split(':', 1)[-1].strip()
+                elif 'Hardware UUID:' in line:
+                    hw_uuid = line.split(':', 1)[-1].strip()
+        except Exception:
+            pass
+    else:
+        # get Linux machine info
+        try:
+            # Getting SMBIOS data from sysfs.
+            # SMBIOS 2.8 present.
+            # argo-1        | 2024-10-01T10:40:51Z ERR  error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:8000                                                                               │
+            # Handle 0x0100, DMI type 1, 27 bytes
+            # System Information
+            #         Manufacturer: DigitalOcean
+            #         Product Name: Droplet
+            #         Serial Number: 411922099
+            #         UUID: fb65f41c-ec24-4539-beaf-f941903bdb2c
+            #         ...
+            #         Family: DigitalOcean_Droplet
+            dmidecode = subprocess.run(['dmidecode', '-t', 'system'], capture_output=True, text=True, check=True)
+            for line in dmidecode.stdout.split('\n'):
+                if 'Manufacturer:' in line:
+                    hw_manufacturer = line.split(':', 1)[-1].strip()
+                elif 'Product Name:' in line:
+                    hw_product = line.split(':', 1)[-1].strip()
+                elif 'UUID:' in line:
+                    hw_uuid = line.split(':', 1)[-1].strip()
+        except Exception:
+            pass
+
+    # Check for VM fingerprint in manufacturer/product name
+    if 'qemu' in hw_product.lower() or 'vbox' in hw_product.lower() or 'lxc' in hw_product.lower() or 'vm' in hw_product.lower():
+        hw_in_vm = True
+    
+    # Check for QEMU explicitly in pmap output
+    try:
+        result = subprocess.run(['pmap', '1'], capture_output=True, text=True, check=True)
+        if 'qemu' in result.stdout.lower():
+            hw_in_vm = True
+    except Exception:
+        pass
+
+    return {
+        "hw_in_docker": hw_in_docker,
+        "hw_in_vm": hw_in_vm,
+        "hw_manufacturer": hw_manufacturer,
+        "hw_product": hw_product,
+        "hw_uuid": hw_uuid,
+    }
+
+def get_public_ip() -> str:
+    def fetch_url(url: str) -> str:
+        with urllib.request.urlopen(url, timeout=5) as response:
+            return response.read().decode('utf-8').strip()
+
+    def fetch_dns(pubip_lookup_host: str) -> str:
+        return socket.gethostbyname(pubip_lookup_host).strip()
+
+    methods = [
+        (lambda: fetch_url("https://ipinfo.io/ip"), lambda r: r),
+        (lambda: fetch_url("https://api.ipify.org?format=json"), lambda r: json.loads(r)['ip']),
+        (lambda: fetch_dns("myip.opendns.com"), lambda r: r),
+        (lambda: fetch_url("http://whatismyip.akamai.com/"), lambda r: r),  # try HTTP as final fallback in case of TLS/system time errors
+    ]
+
+    for fetch, parse in methods:
+        try:
+            result = parse(fetch())
+            if result:
+                return result
+        except Exception:
+            continue
+
+    raise Exception("Could not determine public IP address")
+
+def get_local_ip(remote_ip: str='1.1.1.1', remote_port: int=80) -> str:
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+            s.connect((remote_ip, remote_port))
+            return s.getsockname()[0]
+    except Exception:
+        pass
+    return '127.0.0.1'
+
+ip_addrs = lambda addrs: (a for a in addrs if a.family == socket.AF_INET)
+mac_addrs = lambda addrs: (a for a in addrs if a.family == psutil.AF_LINK)
+
+def get_isp_info(ip=None):
+    # Get public IP
+    try:
+        ip = ip or urllib.request.urlopen('https://api.ipify.org').read().decode('utf8')
+    except Exception:
+        pass
+    
+    # Get ISP name, city, and country
+    data = {}
+    try:
+        url = f'https://ipapi.co/{ip}/json/'
+        response = urllib.request.urlopen(url)
+        data = json.loads(response.read().decode())
+    except Exception:
+        pass
+    
+    isp = data.get('org', 'Unknown')
+    city = data.get('city', 'Unknown')
+    region = data.get('region', 'Unknown')
+    country = data.get('country_name', 'Unknown')
+    
+    # Get system DNS resolver servers
+    dns_server = None
+    try:
+        result = subprocess.run(['dig', 'example.com', 'A'], capture_output=True, text=True, check=True).stdout
+        dns_server = result.split(';; SERVER: ', 1)[-1].split('\n')[0].split('#')[0].strip()
+    except Exception:
+        pass
+    
+    # Get DNS resolver's ISP name
+    # url = f'https://ipapi.co/{dns_server}/json/'
+    # dns_isp = json.loads(urllib.request.urlopen(url).read().decode()).get('org', 'Unknown')
+    
+    return {
+        'isp': isp,
+        'city': city,
+        'region': region,
+        'country': country,
+        'dns_server': dns_server,
+        # 'net_dns_isp': dns_isp,
+    }
+    
+def get_host_network() -> Dict[str, Any]:
+    default_gateway_local_ip = get_local_ip()
+    gateways = psutil.net_if_addrs()
+    
+    for interface, ips in gateways.items():
+        for local_ip in ip_addrs(ips):
+            if default_gateway_local_ip == local_ip.address:
+                mac_address = next(mac_addrs(ips)).address
+                public_ip = get_public_ip()
+                return {
+                    "hostname": max([socket.gethostname(), platform.node()], key=len),
+                    "iface": interface,
+                    "mac_address": mac_address,
+                    "ip_local": local_ip.address,
+                    "ip_public": public_ip,
+                    # "is_behind_nat": local_ip.address != public_ip,
+                    **get_isp_info(public_ip),
+                }
+    
+    raise Exception("Could not determine host network info")
+
+
+def get_os_info() -> Dict[str, Any]:
+    os_release = platform.release()
+    if platform.system().lower() == 'darwin':
+        os_release = 'macOS ' + platform.mac_ver()[0]
+    else:
+        try:
+            os_release = subprocess.run(['lsb_release', '-ds'], capture_output=True, text=True, check=True).stdout.strip()
+        except Exception:
+            pass
+    
+    return {
+        "os_arch": platform.machine(),
+        "os_family": platform.system().lower(),
+        "os_platform": platform.platform(),
+        "os_kernel": platform.version(),
+        "os_release": os_release,
+    }
+
+def get_host_stats() -> Dict[str, Any]:
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_usage = psutil.disk_usage(str(tmp_dir))
+    app_usage = psutil.disk_usage(str(PACKAGE_DIR))
+    data_usage = psutil.disk_usage(str(DATA_DIR))
+    mem_usage = psutil.virtual_memory()
+    swap_usage = psutil.swap_memory()
+    return {
+        "cpu_boot_time": datetime.fromtimestamp(psutil.boot_time()).isoformat(),
+        "cpu_count": psutil.cpu_count(logical=False),
+        "cpu_load": psutil.getloadavg(),
+        # "cpu_pct": psutil.cpu_percent(interval=1),
+        "mem_virt_used_pct": mem_usage.percent,
+        "mem_virt_used_gb": round(mem_usage.used / 1024 / 1024 / 1024, 3),
+        "mem_virt_free_gb": round(mem_usage.free / 1024 / 1024 / 1024, 3),
+        "mem_swap_used_pct": swap_usage.percent,
+        "mem_swap_used_gb": round(swap_usage.used / 1024 / 1024 / 1024, 3),
+        "mem_swap_free_gb": round(swap_usage.free / 1024 / 1024 / 1024, 3),
+        "disk_tmp_used_pct": tmp_usage.percent,
+        "disk_tmp_used_gb": round(tmp_usage.used / 1024 / 1024 / 1024, 3),
+        "disk_tmp_free_gb": round(tmp_usage.free / 1024 / 1024 / 1024, 3),  # in GB
+        "disk_app_used_pct": app_usage.percent,
+        "disk_app_used_gb": round(app_usage.used / 1024 / 1024 / 1024, 3),
+        "disk_app_free_gb": round(app_usage.free / 1024 / 1024 / 1024, 3),
+        "disk_data_used_pct": data_usage.percent,
+        "disk_data_used_gb": round(data_usage.used / 1024 / 1024 / 1024, 3),
+        "disk_data_free_gb": round(data_usage.free / 1024 / 1024 / 1024, 3),
+    }
+
+def get_host_immutable_info(host_info: Dict[str, Any]) -> Dict[str, Any]:
+    return {
+        key: value
+        for key, value in host_info.items()
+        if key in ['guid', 'net_mac', 'os_family', 'cpu_arch']
+    }
+    
+def get_host_guid() -> str:
+    return machineid.hashed_id('archivebox')
+
+# Example usage
+if __name__ == "__main__":
+    host_info = {
+        'guid': get_host_guid(),
+        'os': get_os_info(),
+        'vm': get_vm_info(),
+        'net': get_host_network(),
+        'stats': get_host_stats(),
+    }
+    print(host_info)
+
+# {
+#     'guid': '1cd2dd279f8a854...6943f2384437991a',
+#     'os': {
+#         'os_arch': 'arm64',
+#         'os_family': 'darwin',
+#         'os_platform': 'macOS-14.6.1-arm64-arm-64bit',
+#         'os_kernel': 'Darwin Kernel Version 23.6.0: Mon Jul 29 21:14:30 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6000',
+#         'os_release': 'macOS 14.6.1'
+#     },
+#     'vm': {'hw_in_docker': False, 'hw_in_vm': False, 'hw_manufacturer': 'Apple', 'hw_product': 'Mac Studio Mac13,1', 'hw_uuid': '39A12B50-...-...-...-...'},
+#     'net': {
+#         'hostname': 'somehost.sub.example.com',
+#         'iface': 'en0',
+#         'mac_address': 'ab:cd:ef:12:34:56',
+#         'ip_local': '192.168.2.18',
+#         'ip_public': '123.123.123.123',
+#         'isp': 'AS-SONICTELECOM',
+#         'city': 'Berkeley',
+#         'region': 'California',
+#         'country': 'United States',
+#         'dns_server': '192.168.1.1'
+#     },
+#     'stats': {
+#         'cpu_boot_time': '2024-09-24T21:20:16',
+#         'cpu_count': 10,
+#         'cpu_load': (2.35693359375, 4.013671875, 4.1171875),
+#         'mem_virt_used_pct': 66.0,
+#         'mem_virt_used_gb': 15.109,
+#         'mem_virt_free_gb': 0.065,
+#         'mem_swap_used_pct': 89.4,
+#         'mem_swap_used_gb': 8.045,
+#         'mem_swap_free_gb': 0.955,
+#         'disk_tmp_used_pct': 26.0,
+#         'disk_tmp_used_gb': 113.1,
+#         'disk_tmp_free_gb': 322.028,
+#         'disk_app_used_pct': 56.1,
+#         'disk_app_used_gb': 2138.796,
+#         'disk_app_free_gb': 1675.996,
+#         'disk_data_used_pct': 56.1,
+#         'disk_data_used_gb': 2138.796,
+#         'disk_data_free_gb': 1675.996
+#     }
+# }

archivebox/machine/migrations/0001_initial.py

@@ -0,0 +1,144 @@
+# Generated by Django 5.1.1 on 2024-10-02 04:34
+
+import archivebox.abid_utils.models
+import charidfield.fields
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Machine",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=None,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                        unique=True,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "abid",
+                    charidfield.fields.CharIDField(
+                        blank=True,
+                        db_index=True,
+                        default=None,
+                        help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+                        max_length=30,
+                        null=True,
+                        prefix="mxn_",
+                        unique=True,
+                    ),
+                ),
+                (
+                    "created_at",
+                    archivebox.abid_utils.models.AutoDateTimeField(
+                        db_index=True, default=None
+                    ),
+                ),
+                ("modified_at", models.DateTimeField(auto_now=True)),
+                (
+                    "guid",
+                    models.CharField(
+                        default=None, editable=False, max_length=64, unique=True
+                    ),
+                ),
+                ("hostname", models.CharField(default=None, max_length=63)),
+                ("hw_in_docker", models.BooleanField(default=False)),
+                ("hw_in_vm", models.BooleanField(default=False)),
+                ("hw_manufacturer", models.CharField(default=None, max_length=63)),
+                ("hw_product", models.CharField(default=None, max_length=63)),
+                ("hw_uuid", models.CharField(default=None, max_length=255)),
+                ("os_arch", models.CharField(default=None, max_length=15)),
+                ("os_family", models.CharField(default=None, max_length=15)),
+                ("os_platform", models.CharField(default=None, max_length=63)),
+                ("os_release", models.CharField(default=None, max_length=63)),
+                ("os_kernel", models.CharField(default=None, max_length=255)),
+                ("stats", models.JSONField(default=None)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="NetworkInterface",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=None,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                        unique=True,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "abid",
+                    charidfield.fields.CharIDField(
+                        blank=True,
+                        db_index=True,
+                        default=None,
+                        help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+                        max_length=30,
+                        null=True,
+                        prefix="ixf_",
+                        unique=True,
+                    ),
+                ),
+                (
+                    "created_at",
+                    archivebox.abid_utils.models.AutoDateTimeField(
+                        db_index=True, default=None
+                    ),
+                ),
+                ("modified_at", models.DateTimeField(auto_now=True)),
+                (
+                    "mac_address",
+                    models.CharField(default=None, editable=False, max_length=17),
+                ),
+                (
+                    "ip_public",
+                    models.GenericIPAddressField(default=None, editable=False),
+                ),
+                (
+                    "ip_local",
+                    models.GenericIPAddressField(default=None, editable=False),
+                ),
+                (
+                    "dns_server",
+                    models.GenericIPAddressField(default=None, editable=False),
+                ),
+                ("iface", models.CharField(default=None, max_length=15)),
+                ("hostname", models.CharField(default=None, max_length=63)),
+                ("isp", models.CharField(default=None, max_length=63)),
+                ("city", models.CharField(default=None, max_length=63)),
+                ("region", models.CharField(default=None, max_length=63)),
+                ("country", models.CharField(default=None, max_length=63)),
+                (
+                    "machine",
+                    models.ForeignKey(
+                        default=None,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="machine.machine",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {
+                    ("machine", "ip_public", "ip_local", "mac_address", "dns_server")
+                },
+            },
+        ),
+    ]

archivebox/machine/models.py

@@ -0,0 +1,167 @@
+__package__ = 'archivebox.machine'
+
+import socket
+
+from django.db import models
+from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
+
+from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
+
+CURRENT_MACHINE = None
+CURRENT_INTERFACE = None
+
+class MachineManager(models.Manager):
+    def current(self) -> 'Machine':
+        global CURRENT_MACHINE
+        if CURRENT_MACHINE:
+            return CURRENT_MACHINE
+        
+        guid = get_host_guid()
+        try:
+            CURRENT_MACHINE = self.get(guid=guid)
+            return CURRENT_MACHINE
+        except self.model.DoesNotExist:
+            pass
+        
+        CURRENT_MACHINE = self.model(
+            guid=guid,
+            hostname=socket.gethostname(),
+            **get_os_info(),
+            **get_vm_info(),
+            stats=get_host_stats(),
+        )
+        CURRENT_MACHINE.save()
+        return CURRENT_MACHINE
+
+class Machine(ABIDModel):
+    abid_prefix = 'mxn_'
+    abid_ts_src = 'self.created_at'
+    abid_uri_src = 'self.guid'
+    abid_subtype_src = '"01"'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = False
+
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+
+    # IMMUTABLE PROPERTIES
+    guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
+    
+    # MUTABLE PROPERTIES
+    hostname = models.CharField(max_length=63, default=None, null=False)
+    
+    hw_in_docker = models.BooleanField(default=False, null=False)
+    hw_in_vm = models.BooleanField(default=False, null=False)
+    hw_manufacturer = models.CharField(max_length=63, default=None, null=False) # e.g. Apple
+    hw_product = models.CharField(max_length=63, default=None, null=False)      # e.g. Mac Studio Mac13,1
+    hw_uuid = models.CharField(max_length=255, default=None, null=False)        # e.g. 39A12B50-...-...-...-...
+    
+    os_arch = models.CharField(max_length=15, default=None, null=False)         # e.g. arm64
+    os_family = models.CharField(max_length=15, default=None, null=False)       # e.g. darwin
+    os_platform = models.CharField(max_length=63, default=None, null=False)     # e.g. macOS-14.6.1-arm64-arm-64bit
+    os_release = models.CharField(max_length=63, default=None, null=False)      # e.g. macOS 14.6.1
+    os_kernel = models.CharField(max_length=255, default=None, null=False)      # e.g. Darwin Kernel Version 23.6.0: Mon Jul 29 21:14:30 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6000
+    
+    stats = models.JSONField(default=None, null=False)
+    
+    objects = MachineManager()
+    
+    networkinterface_set: models.Manager['NetworkInterface']
+
+
+class NetworkInterfaceManager(models.Manager):
+    def current(self) -> 'NetworkInterface':
+        global CURRENT_INTERFACE
+        if CURRENT_INTERFACE:
+            return CURRENT_INTERFACE
+        
+        machine = Machine.objects.current()
+        net_info = get_host_network()
+        try:
+            CURRENT_INTERFACE = self.get(
+                machine=machine,
+                ip_public=net_info['ip_public'],
+                ip_local=net_info['ip_local'],
+                mac_address=net_info['mac_address'],
+                dns_server=net_info['dns_server'],
+            )
+            return CURRENT_INTERFACE
+        except self.model.DoesNotExist:
+            pass
+        
+        CURRENT_INTERFACE = self.model(
+            machine=machine,
+            **get_host_network(),
+        )
+        CURRENT_INTERFACE.save()
+        return CURRENT_INTERFACE
+            
+
+
+class NetworkInterface(ABIDModel):
+    abid_prefix = 'ixf_'
+    abid_ts_src = 'self.machine.created_at'
+    abid_uri_src = 'self.machine.guid'
+    abid_subtype_src = 'self.iface'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = False
+    
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    
+    machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
+
+    # IMMUTABLE PROPERTIES
+    mac_address = models.CharField(max_length=17, default=None, null=False, editable=False)   # e.g. ab:cd:ef:12:34:56
+    ip_public = models.GenericIPAddressField(default=None, null=False, editable=False)        # e.g. 123.123.123.123 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+    ip_local = models.GenericIPAddressField(default=None, null=False, editable=False)         # e.g. 192.168.2.18    or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+    dns_server = models.GenericIPAddressField(default=None, null=False, editable=False)       # e.g. 8.8.8.8         or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+    
+    # MUTABLE PROPERTIES
+    iface = models.CharField(max_length=15, default=None, null=False)                         # e.g. en0
+    hostname = models.CharField(max_length=63, default=None, null=False)                      # e.g. somehost.sub.example.com
+    isp = models.CharField(max_length=63, default=None, null=False)                           # e.g. AS-SONICTELECOM
+    city = models.CharField(max_length=63, default=None, null=False)                          # e.g. Berkeley
+    region = models.CharField(max_length=63, default=None, null=False)                        # e.g. California
+    country = models.CharField(max_length=63, default=None, null=False)                       # e.g. United States
+
+    objects = NetworkInterfaceManager()
+    
+    class Meta:
+        unique_together = (
+            ('machine', 'ip_public', 'ip_local', 'mac_address', 'dns_server'),
+        )
+        
+
+# class InstalledBinary(ABIDModel):
+#     abid_prefix = 'bin_'
+#     abid_ts_src = 'self.machine.created_at'
+#     abid_uri_src = 'self.machine.guid'
+#     abid_subtype_src = 'self.binprovider'
+#     abid_rand_src = 'self.id'
+#     abid_drift_allowed = False
+    
+#     id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+#     abid = ABIDField(prefix=abid_prefix)
+
+#     created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+#     modified_at = models.DateTimeField(auto_now=True)
+    
+#     machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
+#     binprovider = models.CharField(max_length=255, default=None, null=False)
+    
+#     name = models.CharField(max_length=255, default=None, null=False)
+#     version = models.CharField(max_length=255, default=None, null=False)
+#     abspath = models.CharField(max_length=255, default=None, null=False)
+#     sha256 = models.CharField(max_length=255, default=None, null=False)
+    
+#     class Meta:
+#         unique_together = (
+#             ('machine', 'binprovider', 'version', 'abspath', 'sha256'),
+#         )

pdm.lock

@@ -5,7 +5,7 @@
 groups = ["default", "all", "ldap", "sonic"]
 strategy = ["inherit_metadata"]
 lock_version = "4.5.0"
-content_hash = "sha256:cdf785c77dcdb8927b7743c36374dc5f2377db78622d27eb8356648d61275a0a"
+content_hash = "sha256:3c924966bd7b6d20a3e653f83b72f7c4160088f136e0d4621650c96b23f75803"
 
 [[metadata.targets]]
 requires_python = "==3.11.*"
@@ -13,7 +13,7 @@ platform = "manylinux_2_17_x86_64"
 
 [[metadata.targets]]
 requires_python = "==3.11.*"
-platform = "macos_12_0_arm64"
+platform = "macos_14_0_arm64"
 
 [[package]]
 name = "annotated-types"
@@ -713,7 +713,7 @@ files = [
 
 [[package]]
 name = "httpcore"
-version = "1.0.5"
+version = "1.0.6"
 requires_python = ">=3.8"
 summary = "A minimal low-level HTTP client."
 groups = ["default"]
@@ -723,8 +723,8 @@ dependencies = [
     "h11<0.15,>=0.13",
 ]
 files = [
-    {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
+    {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"},
-    {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
+    {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"},
 ]
 
 [[package]]
@@ -842,12 +842,12 @@ files = [
 
 [[package]]
 name = "mailchecker"
-version = "6.0.9"
+version = "6.0.10"
 summary = "Cross-language email validation. Backed by a database of thousands throwable email providers."
 groups = ["default"]
 marker = "python_version == \"3.11\""
 files = [
-    {file = "mailchecker-6.0.9.tar.gz", hash = "sha256:f17e907ffe6f6faedc243f57eb0c9c951f61dec9af8e96922c1dcd093389b88d"},
+    {file = "mailchecker-6.0.10.tar.gz", hash = "sha256:d933fecb90a66459c8aa543a272890f97c02f6cbf30a3f5016ce2a1699848bee"},
 ]
 
 [[package]]
@@ -993,19 +993,6 @@ dependencies = [
     "requests",
 ]
 
-[[package]]
-name = "pocket"
-version = "0.3.7"
-git = "https://github.com/tapanpandita/pocket.git"
-ref = "v0.3.7"
-revision = "5a144438cc89bfc0ec94db960718ccf1f76468c1"
-summary = "api wrapper for getpocket.com"
-groups = ["default"]
-marker = "python_version == \"3.11\""
-dependencies = [
-    "requests",
-]
-
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.48"
@@ -1056,6 +1043,20 @@ files = [
     {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"},
 ]
 
+[[package]]
+name = "py-machineid"
+version = "0.6.0"
+summary = "Get the unique machine ID of any host (without admin privileges)"
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "winregistry; sys_platform == \"win32\"",
+]
+files = [
+    {file = "py-machineid-0.6.0.tar.gz", hash = "sha256:00c38d8521d429a4539bdd92967234db28a1a2b4b263062b351ca002332e633f"},
+    {file = "py_machineid-0.6.0-py3-none-any.whl", hash = "sha256:63214f8a98737311716b29d279716dc121a6495f16486caf5c032433f81cdfd6"},
+]
+
 [[package]]
 name = "pyasn1"
 version = "0.6.1"
@@ -1144,7 +1145,7 @@ files = [
 
 [[package]]
 name = "pydantic-pkgr"
-version = "0.3.7"
+version = "0.3.8"
 requires_python = ">=3.10"
 summary = "System package manager APIs in strongly typed Python"
 groups = ["default"]
@@ -1155,8 +1156,8 @@ dependencies = [
     "typing-extensions>=4.11.0",
 ]
 files = [
-    {file = "pydantic_pkgr-0.3.7-py3-none-any.whl", hash = "sha256:fdb63b2cee79d7c9d53673b9d61afa846921fd4950a8c16a8c4d2555cd0f6478"},
+    {file = "pydantic_pkgr-0.3.8-py3-none-any.whl", hash = "sha256:fefa34449feb8fc09d73d6beb8a61afe5959b1a848f0a5bba9db1d092d7099be"},
-    {file = "pydantic_pkgr-0.3.7.tar.gz", hash = "sha256:6e575cdc3584d375eb8d5024e5e8bade1c225c2aee3af1a076951dbc1a2c1f2d"},
+    {file = "pydantic_pkgr-0.3.8.tar.gz", hash = "sha256:5ca12f4ee1c82ce0a2231c36b898534899a40a9e77cc4c97175fac9d1dc6e351"},
 ]
 
 [[package]]
@@ -1427,19 +1428,19 @@ files = [
 
 [[package]]
 name = "rich"
-version = "13.8.1"
+version = "13.9.1"
-requires_python = ">=3.7.0"
+requires_python = ">=3.8.0"
 summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
 groups = ["default"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "markdown-it-py>=2.2.0",
     "pygments<3.0.0,>=2.13.0",
-    "typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"",
+    "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"",
 ]
 files = [
-    {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"},
+    {file = "rich-13.9.1-py3-none-any.whl", hash = "sha256:b340e739f30aa58921dc477b8adaa9ecdb7cecc217be01d93730ee1bc8aa83be"},
-    {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"},
+    {file = "rich-13.9.1.tar.gz", hash = "sha256:097cffdf85db1babe30cc7deba5ab3a29e1b9885047dab24c57e9a7f8a9c1466"},
 ]
 
 [[package]]
@@ -1815,7 +1816,7 @@ files = [
 
 [[package]]
 name = "yt-dlp"
-version = "2024.8.6"
+version = "2024.9.27"
 requires_python = ">=3.8"
 summary = "A feature-rich command-line audio/video downloader"
 groups = ["default"]
@@ -1828,11 +1829,11 @@ dependencies = [
     "pycryptodomex",
     "requests<3,>=2.32.2",
     "urllib3<3,>=1.26.17",
-    "websockets>=12.0",
+    "websockets>=13.0",
 ]
 files = [
-    {file = "yt_dlp-2024.8.6-py3-none-any.whl", hash = "sha256:ab507ff600bd9269ad4d654e309646976778f0e243eaa2f6c3c3214278bb2922"},
+    {file = "yt_dlp-2024.9.27-py3-none-any.whl", hash = "sha256:2717468dd697fcfcf9a89f493ba30a3830cdfb276c09750e5b561b08b9ef5f69"},
-    {file = "yt_dlp-2024.8.6.tar.gz", hash = "sha256:e8551f26bc8bf67b99c12373cc87ed2073436c3437e53290878d0f4b4bb1f663"},
+    {file = "yt_dlp-2024.9.27.tar.gz", hash = "sha256:86605542e17e2e23ad23145b637ec308133762a15a5dedac4ae50b7973237026"},
 ]
 
 [[package]]

pyproject.toml

@@ -86,6 +86,7 @@ dependencies = [
     "base32-crockford==0.3.0",
     ############# Extractor Dependencies #############
     "yt-dlp>=2024.8.6",               # for: media
+    "py-machineid>=0.6.0",
 ]
 
 # pdm lock --group=':all' 

requirements.txt

@@ -46,7 +46,7 @@ executing==2.1.0; python_version == "3.11"
 feedparser==6.0.11; python_version == "3.11"
 ftfy==6.2.3; python_version == "3.11"
 h11==0.14.0; python_version == "3.11"
-httpcore==1.0.5; python_version == "3.11"
+httpcore==1.0.6; python_version == "3.11"
 httpx==0.27.2; python_version == "3.11"
 huey==2.5.2; python_version == "3.11"
 hyperlink==21.0.0; python_version == "3.11"
@@ -54,7 +54,7 @@ idna==3.10; python_version == "3.11"
 incremental==24.7.2; python_version == "3.11"
 ipython==8.27.0; python_version == "3.11"
 jedi==0.19.1; python_version == "3.11"
-mailchecker==6.0.9; python_version == "3.11"
+mailchecker==6.0.10; python_version == "3.11"
 markdown-it-py==3.0.0; python_version == "3.11"
 matplotlib-inline==0.1.7; python_version == "3.11"
 mdurl==0.1.2; python_version == "3.11"
@@ -76,7 +76,7 @@ pycparser==2.22; platform_python_implementation != "PyPy" and python_version ==
 pycryptodomex==3.20.0; python_version == "3.11"
 pydantic==2.9.2; python_version == "3.11"
 pydantic-core==2.23.4; python_version == "3.11"
-pydantic-pkgr==0.3.7; python_version == "3.11"
+pydantic-pkgr==0.3.8; python_version == "3.11"
 pydantic-settings==2.5.2; python_version == "3.11"
 pygments==2.18.0; python_version == "3.11"
 pyopenssl==24.2.1; python_version == "3.11"
@@ -94,7 +94,7 @@ pytz==2024.2; python_version == "3.11"
 pyyaml==6.0.2; python_version == "3.11"
 regex==2024.9.11; python_version == "3.11"
 requests==2.32.3; python_version == "3.11"
-rich==13.8.1; python_version == "3.11"
+rich==13.9.1; python_version == "3.11"
 service-identity==24.1.0; python_version == "3.11"
 setuptools==75.1.0; python_version == "3.11"
 sgmllib3k==1.0.0; python_version == "3.11"
@@ -122,5 +122,5 @@ wcwidth==0.2.13; python_version == "3.11"
 websockets==13.1; python_version == "3.11"
 xlrd==2.0.1; python_version == "3.11"
 xmltodict==0.13.0; python_version == "3.11"
-yt-dlp==2024.8.6; python_version == "3.11"
+yt-dlp==2024.9.27; python_version == "3.11"
 zope-interface==7.0.3; python_version == "3.11"