Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 04:03:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

add py-machineid lib for new machine app

Browse source
This commit is contained in:
Nick Sweeting 2024-10-01 21:46:35 -07:00
parent 4a19051f4a
commit f46d62a114
No known key found for this signature in database
10 changed files with 728 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
archivebox/core/admin.py
View file

@ -29,6 +29,7 @@ from core.mixins import SearchResultsAdminMixin
from api.models import APIToken
from abid_utils.admin import ABIDModelAdmin
from queues.tasks import bg_archive_links, bg_add
from machine.models import Machine, NetworkInterface
from index.html import snapshot_icons
from logging_util import printable_filesize
@ -778,3 +779,53 @@ class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
@admin.register(Machine, site=archivebox_admin)
class MachineAdmin(ABIDModelAdmin):
list_display = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
sort_fields = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
# search_fields = ('id', 'abid', 'guid', 'hostname', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release')
readonly_fields = ('guid', 'created_at', 'modified_at', 'abid_info', 'ips')
fields = (*readonly_fields, 'hostname', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release', 'stats')
list_filter = ('hw_in_docker', 'hw_in_vm', 'os_arch', 'os_family', 'os_platform')
ordering = ['-created_at']
list_per_page = 100
@admin.display(
description='Public IP',
ordering='networkinterface__ip_public',
)
def ips(self, machine):
return format_html(
'<a href="/admin/machine/networkinterface/?q={}"><b><code>{}</code></b></a>',
machine.abid,
', '.join(machine.networkinterface_set.values_list('ip_public', flat=True)),
)
@admin.register(NetworkInterface, site=archivebox_admin)
class NetworkInterfaceAdmin(ABIDModelAdmin):
list_display = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
sort_fields = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
search_fields = ('abid', 'machine__abid', 'iface', 'ip_public', 'ip_local', 'mac_address', 'dns_server', 'hostname', 'isp', 'city', 'region', 'country')
readonly_fields = ('machine', 'created_at', 'modified_at', 'abid_info', 'mac_address', 'ip_public', 'ip_local', 'dns_server')
fields = (*readonly_fields, 'iface', 'hostname', 'isp', 'city', 'region', 'country')
list_filter = ('isp', 'country', 'region')
ordering = ['-created_at']
list_per_page = 100
@admin.display(
description='Machine',
ordering='machine__abid',
)
def machine_info(self, iface):
return format_html(
'<a href="/admin/machine/machine/{}/change"><b><code>[{}]</code></b> &nbsp; {}</a>',
iface.machine.id,
iface.machine.abid,
iface.machine.hostname,
)

3
archivebox/core/settings.py
View file

@ -98,7 +98,8 @@ INSTALLED_APPS = [
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
# Our ArchiveBox-provided apps
#'config', # ArchiveBox config settings
#'config', # ArchiveBox config settings (loaded as a plugin, don't need to add it here)
'machine', # handles collecting and storing information about the host machine, network interfaces, installed binaries, etc.
'queues', # handles starting and managing background workers and processes
'abid_utils', # handles ABID ID creation, handling, and models
'core', # core django model with Snapshot, ArchiveResult, etc.

8
archivebox/machine/apps.py Normal file
View file

@ -0,0 +1,8 @@
from django.apps import AppConfig
class MachineConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'machine'
verbose_name = 'Machine Info'

317
archivebox/machine/detect.py Normal file
View file

@ -0,0 +1,317 @@
import os
import json
import socket
import urllib.request
from typing import Dict, Any
from pathlib import Path
import subprocess
import platform
import tempfile
from datetime import datetime
import psutil
import machineid # https://github.com/keygen-sh/py-machineid
from rich import print
PACKAGE_DIR = Path(__file__).parent
DATA_DIR = Path('.').resolve()
def get_vm_info():
hw_in_docker = bool(os.getenv('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE'))
hw_in_vm = False
try:
# check for traces of docker/containerd/podman in cgroup
with open('/proc/self/cgroup', 'r') as procfile:
for line in procfile:
cgroup = line.strip() # .split('/', 1)[-1].lower()
if 'docker' in cgroup or 'containerd' in cgroup or 'podman' in cgroup:
hw_in_docker = True
except Exception:
pass
hw_manufacturer = 'Docker' if hw_in_docker else 'Unknown'
hw_product = 'Container' if hw_in_docker else 'Unknown'
hw_uuid = machineid.id()
if platform.system().lower() == 'darwin':
# Get macOS machine info
hw_manufacturer = 'Apple'
hw_product = 'Mac'
try:
# Hardware:
# Hardware Overview:
# Model Name: Mac Studio
# Model Identifier: Mac13,1
# Model Number: MJMV3LL/A
# ...
# Serial Number (system): M230YYTD77
# Hardware UUID: 39A12B50-1972-5910-8BEE-235AD20C8EE3
# ...
result = subprocess.run(['system_profiler', 'SPHardwareDataType'], capture_output=True, text=True, check=True)
for line in result.stdout.split('\n'):
if 'Model Name:' in line:
hw_product = line.split(':', 1)[-1].strip()
elif 'Model Identifier:' in line:
hw_product += ' ' + line.split(':', 1)[-1].strip()
elif 'Hardware UUID:' in line:
hw_uuid = line.split(':', 1)[-1].strip()
except Exception:
pass
else:
# get Linux machine info
try:
# Getting SMBIOS data from sysfs.
# SMBIOS 2.8 present.
# argo-1 | 2024-10-01T10:40:51Z ERR error="Incoming request ended abruptly: context canceled" connIndex=2 event=1 ingressRule=0 originService=http://archivebox:8000 │
# Handle 0x0100, DMI type 1, 27 bytes
# System Information
# Manufacturer: DigitalOcean
# Product Name: Droplet
# Serial Number: 411922099
# UUID: fb65f41c-ec24-4539-beaf-f941903bdb2c
# ...
# Family: DigitalOcean_Droplet
dmidecode = subprocess.run(['dmidecode', '-t', 'system'], capture_output=True, text=True, check=True)
for line in dmidecode.stdout.split('\n'):
if 'Manufacturer:' in line:
hw_manufacturer = line.split(':', 1)[-1].strip()
elif 'Product Name:' in line:
hw_product = line.split(':', 1)[-1].strip()
elif 'UUID:' in line:
hw_uuid = line.split(':', 1)[-1].strip()
except Exception:
pass
# Check for VM fingerprint in manufacturer/product name
if 'qemu' in hw_product.lower() or 'vbox' in hw_product.lower() or 'lxc' in hw_product.lower() or 'vm' in hw_product.lower():
hw_in_vm = True
# Check for QEMU explicitly in pmap output
try:
result = subprocess.run(['pmap', '1'], capture_output=True, text=True, check=True)
if 'qemu' in result.stdout.lower():
hw_in_vm = True
except Exception:
pass
return {
"hw_in_docker": hw_in_docker,
"hw_in_vm": hw_in_vm,
"hw_manufacturer": hw_manufacturer,
"hw_product": hw_product,
"hw_uuid": hw_uuid,
}
def get_public_ip() -> str:
def fetch_url(url: str) -> str:
with urllib.request.urlopen(url, timeout=5) as response:
return response.read().decode('utf-8').strip()
def fetch_dns(pubip_lookup_host: str) -> str:
return socket.gethostbyname(pubip_lookup_host).strip()
methods = [
(lambda: fetch_url("https://ipinfo.io/ip"), lambda r: r),
(lambda: fetch_url("https://api.ipify.org?format=json"), lambda r: json.loads(r)['ip']),
(lambda: fetch_dns("myip.opendns.com"), lambda r: r),
(lambda: fetch_url("http://whatismyip.akamai.com/"), lambda r: r), # try HTTP as final fallback in case of TLS/system time errors
]
for fetch, parse in methods:
try:
result = parse(fetch())
if result:
return result
except Exception:
continue
raise Exception("Could not determine public IP address")
def get_local_ip(remote_ip: str='1.1.1.1', remote_port: int=80) -> str:
try:
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
s.connect((remote_ip, remote_port))
return s.getsockname()[0]
except Exception:
pass
return '127.0.0.1'
ip_addrs = lambda addrs: (a for a in addrs if a.family == socket.AF_INET)
mac_addrs = lambda addrs: (a for a in addrs if a.family == psutil.AF_LINK)
def get_isp_info(ip=None):
# Get public IP
try:
ip = ip or urllib.request.urlopen('https://api.ipify.org').read().decode('utf8')
except Exception:
pass
# Get ISP name, city, and country
data = {}
try:
url = f'https://ipapi.co/{ip}/json/'
response = urllib.request.urlopen(url)
data = json.loads(response.read().decode())
except Exception:
pass
isp = data.get('org', 'Unknown')
city = data.get('city', 'Unknown')
region = data.get('region', 'Unknown')
country = data.get('country_name', 'Unknown')
# Get system DNS resolver servers
dns_server = None
try:
result = subprocess.run(['dig', 'example.com', 'A'], capture_output=True, text=True, check=True).stdout
dns_server = result.split(';; SERVER: ', 1)[-1].split('\n')[0].split('#')[0].strip()
except Exception:
pass
# Get DNS resolver's ISP name
# url = f'https://ipapi.co/{dns_server}/json/'
# dns_isp = json.loads(urllib.request.urlopen(url).read().decode()).get('org', 'Unknown')
return {
'isp': isp,
'city': city,
'region': region,
'country': country,
'dns_server': dns_server,
# 'net_dns_isp': dns_isp,
}
def get_host_network() -> Dict[str, Any]:
default_gateway_local_ip = get_local_ip()
gateways = psutil.net_if_addrs()
for interface, ips in gateways.items():
for local_ip in ip_addrs(ips):
if default_gateway_local_ip == local_ip.address:
mac_address = next(mac_addrs(ips)).address
public_ip = get_public_ip()
return {
"hostname": max([socket.gethostname(), platform.node()], key=len),
"iface": interface,
"mac_address": mac_address,
"ip_local": local_ip.address,
"ip_public": public_ip,
# "is_behind_nat": local_ip.address != public_ip,
**get_isp_info(public_ip),
}
raise Exception("Could not determine host network info")
def get_os_info() -> Dict[str, Any]:
os_release = platform.release()
if platform.system().lower() == 'darwin':
os_release = 'macOS ' + platform.mac_ver()[0]
else:
try:
os_release = subprocess.run(['lsb_release', '-ds'], capture_output=True, text=True, check=True).stdout.strip()
except Exception:
pass
return {
"os_arch": platform.machine(),
"os_family": platform.system().lower(),
"os_platform": platform.platform(),
"os_kernel": platform.version(),
"os_release": os_release,
}
def get_host_stats() -> Dict[str, Any]:
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_usage = psutil.disk_usage(str(tmp_dir))
app_usage = psutil.disk_usage(str(PACKAGE_DIR))
data_usage = psutil.disk_usage(str(DATA_DIR))
mem_usage = psutil.virtual_memory()
swap_usage = psutil.swap_memory()
return {
"cpu_boot_time": datetime.fromtimestamp(psutil.boot_time()).isoformat(),
"cpu_count": psutil.cpu_count(logical=False),
"cpu_load": psutil.getloadavg(),
# "cpu_pct": psutil.cpu_percent(interval=1),
"mem_virt_used_pct": mem_usage.percent,
"mem_virt_used_gb": round(mem_usage.used / 1024 / 1024 / 1024, 3),
"mem_virt_free_gb": round(mem_usage.free / 1024 / 1024 / 1024, 3),
"mem_swap_used_pct": swap_usage.percent,
"mem_swap_used_gb": round(swap_usage.used / 1024 / 1024 / 1024, 3),
"mem_swap_free_gb": round(swap_usage.free / 1024 / 1024 / 1024, 3),
"disk_tmp_used_pct": tmp_usage.percent,
"disk_tmp_used_gb": round(tmp_usage.used / 1024 / 1024 / 1024, 3),
"disk_tmp_free_gb": round(tmp_usage.free / 1024 / 1024 / 1024, 3), # in GB
"disk_app_used_pct": app_usage.percent,
"disk_app_used_gb": round(app_usage.used / 1024 / 1024 / 1024, 3),
"disk_app_free_gb": round(app_usage.free / 1024 / 1024 / 1024, 3),
"disk_data_used_pct": data_usage.percent,
"disk_data_used_gb": round(data_usage.used / 1024 / 1024 / 1024, 3),
"disk_data_free_gb": round(data_usage.free / 1024 / 1024 / 1024, 3),
}
def get_host_immutable_info(host_info: Dict[str, Any]) -> Dict[str, Any]:
return {
key: value
for key, value in host_info.items()
if key in ['guid', 'net_mac', 'os_family', 'cpu_arch']
}
def get_host_guid() -> str:
return machineid.hashed_id('archivebox')
# Example usage
if __name__ == "__main__":
host_info = {
'guid': get_host_guid(),
'os': get_os_info(),
'vm': get_vm_info(),
'net': get_host_network(),
'stats': get_host_stats(),
}
print(host_info)
# {
# 'guid': '1cd2dd279f8a854...6943f2384437991a',
# 'os': {
# 'os_arch': 'arm64',
# 'os_family': 'darwin',
# 'os_platform': 'macOS-14.6.1-arm64-arm-64bit',
# 'os_kernel': 'Darwin Kernel Version 23.6.0: Mon Jul 29 21:14:30 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6000',
# 'os_release': 'macOS 14.6.1'
# },
# 'vm': {'hw_in_docker': False, 'hw_in_vm': False, 'hw_manufacturer': 'Apple', 'hw_product': 'Mac Studio Mac13,1', 'hw_uuid': '39A12B50-...-...-...-...'},
# 'net': {
# 'hostname': 'somehost.sub.example.com',
# 'iface': 'en0',
# 'mac_address': 'ab:cd:ef:12:34:56',
# 'ip_local': '192.168.2.18',
# 'ip_public': '123.123.123.123',
# 'isp': 'AS-SONICTELECOM',
# 'city': 'Berkeley',
# 'region': 'California',
# 'country': 'United States',
# 'dns_server': '192.168.1.1'
# },
# 'stats': {
# 'cpu_boot_time': '2024-09-24T21:20:16',
# 'cpu_count': 10,
# 'cpu_load': (2.35693359375, 4.013671875, 4.1171875),
# 'mem_virt_used_pct': 66.0,
# 'mem_virt_used_gb': 15.109,
# 'mem_virt_free_gb': 0.065,
# 'mem_swap_used_pct': 89.4,
# 'mem_swap_used_gb': 8.045,
# 'mem_swap_free_gb': 0.955,
# 'disk_tmp_used_pct': 26.0,
# 'disk_tmp_used_gb': 113.1,
# 'disk_tmp_free_gb': 322.028,
# 'disk_app_used_pct': 56.1,
# 'disk_app_used_gb': 2138.796,
# 'disk_app_free_gb': 1675.996,
# 'disk_data_used_pct': 56.1,
# 'disk_data_used_gb': 2138.796,
# 'disk_data_free_gb': 1675.996
# }
# }

144
archivebox/machine/migrations/0001_initial.py Normal file
View file

@ -0,0 +1,144 @@
# Generated by Django 5.1.1 on 2024-10-02 04:34
import archivebox.abid_utils.models
import charidfield.fields
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Machine",
fields=[
(
"id",
models.UUIDField(
default=None,
editable=False,
primary_key=True,
serialize=False,
unique=True,
verbose_name="ID",
),
),
(
"abid",
charidfield.fields.CharIDField(
blank=True,
db_index=True,
default=None,
help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
max_length=30,
null=True,
prefix="mxn_",
unique=True,
),
),
(
"created_at",
archivebox.abid_utils.models.AutoDateTimeField(
db_index=True, default=None
),
),
("modified_at", models.DateTimeField(auto_now=True)),
(
"guid",
models.CharField(
default=None, editable=False, max_length=64, unique=True
),
),
("hostname", models.CharField(default=None, max_length=63)),
("hw_in_docker", models.BooleanField(default=False)),
("hw_in_vm", models.BooleanField(default=False)),
("hw_manufacturer", models.CharField(default=None, max_length=63)),
("hw_product", models.CharField(default=None, max_length=63)),
("hw_uuid", models.CharField(default=None, max_length=255)),
("os_arch", models.CharField(default=None, max_length=15)),
("os_family", models.CharField(default=None, max_length=15)),
("os_platform", models.CharField(default=None, max_length=63)),
("os_release", models.CharField(default=None, max_length=63)),
("os_kernel", models.CharField(default=None, max_length=255)),
("stats", models.JSONField(default=None)),
],
options={
"abstract": False,
},
),
migrations.CreateModel(
name="NetworkInterface",
fields=[
(
"id",
models.UUIDField(
default=None,
editable=False,
primary_key=True,
serialize=False,
unique=True,
verbose_name="ID",
),
),
(
"abid",
charidfield.fields.CharIDField(
blank=True,
db_index=True,
default=None,
help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
max_length=30,
null=True,
prefix="ixf_",
unique=True,
),
),
(
"created_at",
archivebox.abid_utils.models.AutoDateTimeField(
db_index=True, default=None
),
),
("modified_at", models.DateTimeField(auto_now=True)),
(
"mac_address",
models.CharField(default=None, editable=False, max_length=17),
),
(
"ip_public",
models.GenericIPAddressField(default=None, editable=False),
),
(
"ip_local",
models.GenericIPAddressField(default=None, editable=False),
),
(
"dns_server",
models.GenericIPAddressField(default=None, editable=False),
),
("iface", models.CharField(default=None, max_length=15)),
("hostname", models.CharField(default=None, max_length=63)),
("isp", models.CharField(default=None, max_length=63)),
("city", models.CharField(default=None, max_length=63)),
("region", models.CharField(default=None, max_length=63)),
("country", models.CharField(default=None, max_length=63)),
(
"machine",
models.ForeignKey(
default=None,
on_delete=django.db.models.deletion.CASCADE,
to="machine.machine",
),
),
],
options={
"unique_together": {
("machine", "ip_public", "ip_local", "mac_address", "dns_server")
},
},
),
]

0
archivebox/machine/migrations/__init__.py Normal file
View file

167
archivebox/machine/models.py Normal file
View file

@ -0,0 +1,167 @@
__package__ = 'archivebox.machine'
import socket
from django.db import models
from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
CURRENT_MACHINE = None
CURRENT_INTERFACE = None
class MachineManager(models.Manager):
def current(self) -> 'Machine':
global CURRENT_MACHINE
if CURRENT_MACHINE:
return CURRENT_MACHINE
guid = get_host_guid()
try:
CURRENT_MACHINE = self.get(guid=guid)
return CURRENT_MACHINE
except self.model.DoesNotExist:
pass
CURRENT_MACHINE = self.model(
guid=guid,
hostname=socket.gethostname(),
**get_os_info(),
**get_vm_info(),
stats=get_host_stats(),
)
CURRENT_MACHINE.save()
return CURRENT_MACHINE
class Machine(ABIDModel):
abid_prefix = 'mxn_'
abid_ts_src = 'self.created_at'
abid_uri_src = 'self.guid'
abid_subtype_src = '"01"'
abid_rand_src = 'self.id'
abid_drift_allowed = False
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
created_at = AutoDateTimeField(default=None, null=False, db_index=True)
modified_at = models.DateTimeField(auto_now=True)
# IMMUTABLE PROPERTIES
guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
# MUTABLE PROPERTIES
hostname = models.CharField(max_length=63, default=None, null=False)
hw_in_docker = models.BooleanField(default=False, null=False)
hw_in_vm = models.BooleanField(default=False, null=False)
hw_manufacturer = models.CharField(max_length=63, default=None, null=False) # e.g. Apple
hw_product = models.CharField(max_length=63, default=None, null=False) # e.g. Mac Studio Mac13,1
hw_uuid = models.CharField(max_length=255, default=None, null=False) # e.g. 39A12B50-...-...-...-...
os_arch = models.CharField(max_length=15, default=None, null=False) # e.g. arm64
os_family = models.CharField(max_length=15, default=None, null=False) # e.g. darwin
os_platform = models.CharField(max_length=63, default=None, null=False) # e.g. macOS-14.6.1-arm64-arm-64bit
os_release = models.CharField(max_length=63, default=None, null=False) # e.g. macOS 14.6.1
os_kernel = models.CharField(max_length=255, default=None, null=False) # e.g. Darwin Kernel Version 23.6.0: Mon Jul 29 21:14:30 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T6000
stats = models.JSONField(default=None, null=False)
objects = MachineManager()
networkinterface_set: models.Manager['NetworkInterface']
class NetworkInterfaceManager(models.Manager):
def current(self) -> 'NetworkInterface':
global CURRENT_INTERFACE
if CURRENT_INTERFACE:
return CURRENT_INTERFACE
machine = Machine.objects.current()
net_info = get_host_network()
try:
CURRENT_INTERFACE = self.get(
machine=machine,
ip_public=net_info['ip_public'],
ip_local=net_info['ip_local'],
mac_address=net_info['mac_address'],
dns_server=net_info['dns_server'],
)
return CURRENT_INTERFACE
except self.model.DoesNotExist:
pass
CURRENT_INTERFACE = self.model(
machine=machine,
**get_host_network(),
)
CURRENT_INTERFACE.save()
return CURRENT_INTERFACE
class NetworkInterface(ABIDModel):
abid_prefix = 'ixf_'
abid_ts_src = 'self.machine.created_at'
abid_uri_src = 'self.machine.guid'
abid_subtype_src = 'self.iface'
abid_rand_src = 'self.id'
abid_drift_allowed = False
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
created_at = AutoDateTimeField(default=None, null=False, db_index=True)
modified_at = models.DateTimeField(auto_now=True)
machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
# IMMUTABLE PROPERTIES
mac_address = models.CharField(max_length=17, default=None, null=False, editable=False) # e.g. ab:cd:ef:12:34:56
ip_public = models.GenericIPAddressField(default=None, null=False, editable=False) # e.g. 123.123.123.123 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
ip_local = models.GenericIPAddressField(default=None, null=False, editable=False) # e.g. 192.168.2.18 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
dns_server = models.GenericIPAddressField(default=None, null=False, editable=False) # e.g. 8.8.8.8 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334
# MUTABLE PROPERTIES
iface = models.CharField(max_length=15, default=None, null=False) # e.g. en0
hostname = models.CharField(max_length=63, default=None, null=False) # e.g. somehost.sub.example.com
isp = models.CharField(max_length=63, default=None, null=False) # e.g. AS-SONICTELECOM
city = models.CharField(max_length=63, default=None, null=False) # e.g. Berkeley
region = models.CharField(max_length=63, default=None, null=False) # e.g. California
country = models.CharField(max_length=63, default=None, null=False) # e.g. United States
objects = NetworkInterfaceManager()
class Meta:
unique_together = (
('machine', 'ip_public', 'ip_local', 'mac_address', 'dns_server'),
)
# class InstalledBinary(ABIDModel):
# abid_prefix = 'bin_'
# abid_ts_src = 'self.machine.created_at'
# abid_uri_src = 'self.machine.guid'
# abid_subtype_src = 'self.binprovider'
# abid_rand_src = 'self.id'
# abid_drift_allowed = False
# id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
# abid = ABIDField(prefix=abid_prefix)
# created_at = AutoDateTimeField(default=None, null=False, db_index=True)
# modified_at = models.DateTimeField(auto_now=True)
# machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
# binprovider = models.CharField(max_length=255, default=None, null=False)
# name = models.CharField(max_length=255, default=None, null=False)
# version = models.CharField(max_length=255, default=None, null=False)
# abspath = models.CharField(max_length=255, default=None, null=False)
# sha256 = models.CharField(max_length=255, default=None, null=False)
# class Meta:
# unique_together = (
# ('machine', 'binprovider', 'version', 'abspath', 'sha256'),
# )

65
pdm.lock
View file

@ -5,7 +5,7 @@
groups = ["default", "all", "ldap", "sonic"]
strategy = ["inherit_metadata"]
lock_version = "4.5.0"
content_hash = "sha256:cdf785c77dcdb8927b7743c36374dc5f2377db78622d27eb8356648d61275a0a"
content_hash = "sha256:3c924966bd7b6d20a3e653f83b72f7c4160088f136e0d4621650c96b23f75803"
[[metadata.targets]]
requires_python = "==3.11.*"
@ -13,7 +13,7 @@ platform = "manylinux_2_17_x86_64"
[[metadata.targets]]
requires_python = "==3.11.*"
platform = "macos_12_0_arm64"
platform = "macos_14_0_arm64"
[[package]]
name = "annotated-types"
@ -713,7 +713,7 @@ files = [
[[package]]
name = "httpcore"
version = "1.0.5"
version = "1.0.6"
requires_python = ">=3.8"
summary = "A minimal low-level HTTP client."
groups = ["default"]
@ -723,8 +723,8 @@ dependencies = [
"h11<0.15,>=0.13",
]
files = [
{file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
{file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
{file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"},
{file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"},
]
[[package]]
@ -842,12 +842,12 @@ files = [
[[package]]
name = "mailchecker"
version = "6.0.9"
version = "6.0.10"
summary = "Cross-language email validation. Backed by a database of thousands throwable email providers."
groups = ["default"]
marker = "python_version == \"3.11\""
files = [
{file = "mailchecker-6.0.9.tar.gz", hash = "sha256:f17e907ffe6f6faedc243f57eb0c9c951f61dec9af8e96922c1dcd093389b88d"},
{file = "mailchecker-6.0.10.tar.gz", hash = "sha256:d933fecb90a66459c8aa543a272890f97c02f6cbf30a3f5016ce2a1699848bee"},
]
[[package]]
@ -993,19 +993,6 @@ dependencies = [
"requests",
]
[[package]]
name = "pocket"
version = "0.3.7"
git = "https://github.com/tapanpandita/pocket.git"
ref = "v0.3.7"
revision = "5a144438cc89bfc0ec94db960718ccf1f76468c1"
summary = "api wrapper for getpocket.com"
groups = ["default"]
marker = "python_version == \"3.11\""
dependencies = [
"requests",
]
[[package]]
name = "prompt-toolkit"
version = "3.0.48"
@ -1056,6 +1043,20 @@ files = [
{file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"},
]
[[package]]
name = "py-machineid"
version = "0.6.0"
summary = "Get the unique machine ID of any host (without admin privileges)"
groups = ["default"]
marker = "python_version == \"3.11\""
dependencies = [
"winregistry; sys_platform == \"win32\"",
]
files = [
{file = "py-machineid-0.6.0.tar.gz", hash = "sha256:00c38d8521d429a4539bdd92967234db28a1a2b4b263062b351ca002332e633f"},
{file = "py_machineid-0.6.0-py3-none-any.whl", hash = "sha256:63214f8a98737311716b29d279716dc121a6495f16486caf5c032433f81cdfd6"},
]
[[package]]
name = "pyasn1"
version = "0.6.1"
@ -1144,7 +1145,7 @@ files = [
[[package]]
name = "pydantic-pkgr"
version = "0.3.7"
version = "0.3.8"
requires_python = ">=3.10"
summary = "System package manager APIs in strongly typed Python"
groups = ["default"]
@ -1155,8 +1156,8 @@ dependencies = [
"typing-extensions>=4.11.0",
]
files = [
{file = "pydantic_pkgr-0.3.7-py3-none-any.whl", hash = "sha256:fdb63b2cee79d7c9d53673b9d61afa846921fd4950a8c16a8c4d2555cd0f6478"},
{file = "pydantic_pkgr-0.3.7.tar.gz", hash = "sha256:6e575cdc3584d375eb8d5024e5e8bade1c225c2aee3af1a076951dbc1a2c1f2d"},
{file = "pydantic_pkgr-0.3.8-py3-none-any.whl", hash = "sha256:fefa34449feb8fc09d73d6beb8a61afe5959b1a848f0a5bba9db1d092d7099be"},
{file = "pydantic_pkgr-0.3.8.tar.gz", hash = "sha256:5ca12f4ee1c82ce0a2231c36b898534899a40a9e77cc4c97175fac9d1dc6e351"},
]
[[package]]
@ -1427,19 +1428,19 @@ files = [
[[package]]
name = "rich"
version = "13.8.1"
requires_python = ">=3.7.0"
version = "13.9.1"
requires_python = ">=3.8.0"
summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
groups = ["default"]
marker = "python_version == \"3.11\""
dependencies = [
"markdown-it-py>=2.2.0",
"pygments<3.0.0,>=2.13.0",
"typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"",
"typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"",
]
files = [
{file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"},
{file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"},
{file = "rich-13.9.1-py3-none-any.whl", hash = "sha256:b340e739f30aa58921dc477b8adaa9ecdb7cecc217be01d93730ee1bc8aa83be"},
{file = "rich-13.9.1.tar.gz", hash = "sha256:097cffdf85db1babe30cc7deba5ab3a29e1b9885047dab24c57e9a7f8a9c1466"},
]
[[package]]
@ -1815,7 +1816,7 @@ files = [
[[package]]
name = "yt-dlp"
version = "2024.8.6"
version = "2024.9.27"
requires_python = ">=3.8"
summary = "A feature-rich command-line audio/video downloader"
groups = ["default"]
@ -1828,11 +1829,11 @@ dependencies = [
"pycryptodomex",
"requests<3,>=2.32.2",
"urllib3<3,>=1.26.17",
"websockets>=12.0",
"websockets>=13.0",
]
files = [
{file = "yt_dlp-2024.8.6-py3-none-any.whl", hash = "sha256:ab507ff600bd9269ad4d654e309646976778f0e243eaa2f6c3c3214278bb2922"},
{file = "yt_dlp-2024.8.6.tar.gz", hash = "sha256:e8551f26bc8bf67b99c12373cc87ed2073436c3437e53290878d0f4b4bb1f663"},
{file = "yt_dlp-2024.9.27-py3-none-any.whl", hash = "sha256:2717468dd697fcfcf9a89f493ba30a3830cdfb276c09750e5b561b08b9ef5f69"},
{file = "yt_dlp-2024.9.27.tar.gz", hash = "sha256:86605542e17e2e23ad23145b637ec308133762a15a5dedac4ae50b7973237026"},
]
[[package]]

1
pyproject.toml
View file

@ -86,6 +86,7 @@ dependencies = [
"base32-crockford==0.3.0",
############# Extractor Dependencies #############
"yt-dlp>=2024.8.6", # for: media
"py-machineid>=0.6.0",
]
# pdm lock --group=':all'

10
requirements.txt
View file

@ -46,7 +46,7 @@ executing==2.1.0; python_version == "3.11"
feedparser==6.0.11; python_version == "3.11"
ftfy==6.2.3; python_version == "3.11"
h11==0.14.0; python_version == "3.11"
httpcore==1.0.5; python_version == "3.11"
httpcore==1.0.6; python_version == "3.11"
httpx==0.27.2; python_version == "3.11"
huey==2.5.2; python_version == "3.11"
hyperlink==21.0.0; python_version == "3.11"
@ -54,7 +54,7 @@ idna==3.10; python_version == "3.11"
incremental==24.7.2; python_version == "3.11"
ipython==8.27.0; python_version == "3.11"
jedi==0.19.1; python_version == "3.11"
mailchecker==6.0.9; python_version == "3.11"
mailchecker==6.0.10; python_version == "3.11"
markdown-it-py==3.0.0; python_version == "3.11"
matplotlib-inline==0.1.7; python_version == "3.11"
mdurl==0.1.2; python_version == "3.11"
@ -76,7 +76,7 @@ pycparser==2.22; platform_python_implementation != "PyPy" and python_version ==
pycryptodomex==3.20.0; python_version == "3.11"
pydantic==2.9.2; python_version == "3.11"
pydantic-core==2.23.4; python_version == "3.11"
pydantic-pkgr==0.3.7; python_version == "3.11"
pydantic-pkgr==0.3.8; python_version == "3.11"
pydantic-settings==2.5.2; python_version == "3.11"
pygments==2.18.0; python_version == "3.11"
pyopenssl==24.2.1; python_version == "3.11"
@ -94,7 +94,7 @@ pytz==2024.2; python_version == "3.11"
pyyaml==6.0.2; python_version == "3.11"
regex==2024.9.11; python_version == "3.11"
requests==2.32.3; python_version == "3.11"
rich==13.8.1; python_version == "3.11"
rich==13.9.1; python_version == "3.11"
service-identity==24.1.0; python_version == "3.11"
setuptools==75.1.0; python_version == "3.11"
sgmllib3k==1.0.0; python_version == "3.11"
@ -122,5 +122,5 @@ wcwidth==0.2.13; python_version == "3.11"
websockets==13.1; python_version == "3.11"
xlrd==2.0.1; python_version == "3.11"
xmltodict==0.13.0; python_version == "3.11"
yt-dlp==2024.8.6; python_version == "3.11"
yt-dlp==2024.9.27; python_version == "3.11"
zope-interface==7.0.3; python_version == "3.11"