"""
Miner models: generic web scraping operations and extracted content blocks.
Domain-specific parsers (e.g. BOPV HTML) live in host apps and write into
these models.
"""
import uuid
from django.db import models
from django.db.models import Model, \
UUIDField, CharField, TextField, DateTimeField, JSONField, \
URLField, ForeignKey, CASCADE
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
[docs]
class Scrap(Model):
"""Web scraping operation model."""
id = UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
url = URLField(max_length=255, verbose_name=_('url'))
created_at = DateTimeField(default=timezone.now, verbose_name=_('created at'))
updated_at = DateTimeField(auto_now=True, verbose_name=_('updated at'))
status = CharField(max_length=255, verbose_name=_('status'))
error = TextField(blank=True, null=True, verbose_name=_('error'))
data = JSONField(blank=True, null=True, verbose_name=_('data'))
headers = JSONField(blank=True, null=True, verbose_name=_('headers'))
cookies = JSONField(blank=True, null=True, verbose_name=_('cookies'))
class Meta:
verbose_name = _('scrap')
verbose_name_plural = _('scraps')
ordering = ['-created_at']
def __str__(self):
return f"{self.url} - {self.status}"
[docs]
class Block(Model):
"""Content block extracted from a scraped page."""
id = UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
scrap = ForeignKey(Scrap, on_delete=CASCADE, related_name='blocks', verbose_name=_('scrap'))
content = TextField(blank=True, null=True, verbose_name=_('content'))
metadata = JSONField(blank=True, null=True, verbose_name=_('metadata'))
created_at = DateTimeField(default=timezone.now, verbose_name=_('created at'))
updated_at = DateTimeField(auto_now=True, verbose_name=_('updated at'))
class Meta:
verbose_name = _('block')
verbose_name_plural = _('blocks')
ordering = ['created_at']
def __str__(self):
content_preview = self.content[:50] + '...' if self.content and len(self.content) > 50 else self.content or ''
return f"Block from {self.scrap.url}: {content_preview}"