|
@@ -59,9 +59,6 @@ class SnapshotSearchJob(Job):
|
|
|
< now
|
|
< now
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
- async def run(self, ia_client: InternetArchiveClient):
|
|
|
|
|
- return await ia_client.get_snapshot_id_closest_to(self.collection.url, self.dt)
|
|
|
|
|
-
|
|
|
|
|
|
|
|
|
|
@frozen
|
|
@frozen
|
|
|
class SnapshotFetchJob(Job):
|
|
class SnapshotFetchJob(Job):
|
|
@@ -69,9 +66,6 @@ class SnapshotFetchJob(Job):
|
|
|
collection: ArchiveCollection
|
|
collection: ArchiveCollection
|
|
|
dt: datetime
|
|
dt: datetime
|
|
|
|
|
|
|
|
- async def run(self, ia_client: InternetArchiveClient):
|
|
|
|
|
- return await ia_client.fetch(self.snap_id)
|
|
|
|
|
-
|
|
|
|
|
|
|
|
|
|
@frozen
|
|
@frozen
|
|
|
class SnapshotParseJob(Job):
|
|
class SnapshotParseJob(Job):
|
|
@@ -79,9 +73,6 @@ class SnapshotParseJob(Job):
|
|
|
snapshot: InternetArchiveSnapshot
|
|
snapshot: InternetArchiveSnapshot
|
|
|
dt: datetime
|
|
dt: datetime
|
|
|
|
|
|
|
|
- async def run(self):
|
|
|
|
|
- return await self.collection.MainPageClass.from_snapshot(self.snapshot)
|
|
|
|
|
-
|
|
|
|
|
|
|
|
|
|
@frozen
|
|
@frozen
|
|
|
class SnapshotStoreJob(Job):
|
|
class SnapshotStoreJob(Job):
|
|
@@ -89,9 +80,6 @@ class SnapshotStoreJob(Job):
|
|
|
collection: ArchiveCollection
|
|
collection: ArchiveCollection
|
|
|
dt: datetime
|
|
dt: datetime
|
|
|
|
|
|
|
|
- async def run(self, storage: Storage):
|
|
|
|
|
- return await storage.add_page(self.collection, self.page, self.dt)
|
|
|
|
|
-
|
|
|
|
|
|
|
|
|
|
@frozen
|
|
@frozen
|
|
|
class SearchWorker(Worker):
|
|
class SearchWorker(Worker):
|
|
@@ -111,7 +99,9 @@ class SearchWorker(Worker):
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
try:
|
|
try:
|
|
|
- id_closest = await job.run(self.ia_client)
|
|
|
|
|
|
|
+ id_closest = await self.ia_client.get_snapshot_id_closest_to(
|
|
|
|
|
+ job.collection.url, job.dt
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
delta = job.dt - id_closest.timestamp
|
|
delta = job.dt - id_closest.timestamp
|
|
|
abs_delta = abs(delta)
|
|
abs_delta = abs(delta)
|
|
@@ -152,7 +142,7 @@ class FetchWorker(Worker):
|
|
|
|
|
|
|
|
async def execute(self, job: SnapshotFetchJob):
|
|
async def execute(self, job: SnapshotFetchJob):
|
|
|
try:
|
|
try:
|
|
|
- closest = await job.run(self.ia_client)
|
|
|
|
|
|
|
+ closest = await self.ia_client.fetch(job.snap_id)
|
|
|
return closest, [SnapshotParseJob(job.id_, job.collection, closest, job.dt)]
|
|
return closest, [SnapshotParseJob(job.id_, job.collection, closest, job.dt)]
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
self._log("ERROR", job, f"Error while fetching {job.snap_id}")
|
|
self._log("ERROR", job, f"Error while fetching {job.snap_id}")
|
|
@@ -166,7 +156,7 @@ class ParseWorker(Worker):
|
|
|
|
|
|
|
|
async def execute(self, job: SnapshotParseJob):
|
|
async def execute(self, job: SnapshotParseJob):
|
|
|
try:
|
|
try:
|
|
|
- main_page = await job.run()
|
|
|
|
|
|
|
+ main_page = await job.collection.MainPageClass.from_snapshot(job.snapshot)
|
|
|
return main_page, [
|
|
return main_page, [
|
|
|
SnapshotStoreJob(job.id_, main_page, job.collection, job.dt)
|
|
SnapshotStoreJob(job.id_, main_page, job.collection, job.dt)
|
|
|
]
|
|
]
|
|
@@ -199,7 +189,7 @@ class StoreWorker(Worker):
|
|
|
|
|
|
|
|
async def execute(self, job: SnapshotStoreJob):
|
|
async def execute(self, job: SnapshotStoreJob):
|
|
|
try:
|
|
try:
|
|
|
- return await job.run(self.storage), []
|
|
|
|
|
|
|
+ return await self.storage.add_page(job.collection, job.page, job.dt), []
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
self._log(
|
|
self._log(
|
|
|
"ERROR",
|
|
"ERROR",
|