Use async for data downloading
This commit is contained in:
+11
-15
@@ -1,4 +1,8 @@
|
||||
import typer
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
app = typer.Typer()
|
||||
|
||||
@@ -6,36 +10,28 @@ app = typer.Typer()
|
||||
@app.command()
|
||||
def fetch(file: str, delete_ignore: bool = False):
|
||||
from pathlib import Path
|
||||
from adsbpy.fetch import fetch_lines
|
||||
from adsbpy import fetch
|
||||
import asyncio
|
||||
|
||||
IGNORE_FILE = Path("adsbpy/.ignore")
|
||||
if delete_ignore and IGNORE_FILE.exists():
|
||||
print(f"Deleting ignore file at {IGNORE_FILE}")
|
||||
logger.warning(f"Deleting ignore file at {IGNORE_FILE}")
|
||||
IGNORE_FILE.unlink()
|
||||
ignore_list = []
|
||||
if IGNORE_FILE.exists():
|
||||
print(f"Found ignore file at {IGNORE_FILE}")
|
||||
logger.info(f"Found ignore file at {IGNORE_FILE}")
|
||||
with open(IGNORE_FILE) as f:
|
||||
ignore_list = [line.strip() for line in f.readlines() if line.strip()]
|
||||
else:
|
||||
print("No ignore file found, starting fresh.")
|
||||
logger.info("No ignore file found, starting fresh.")
|
||||
IGNORE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
IGNORE_FILE.touch()
|
||||
|
||||
with open(file) as f:
|
||||
lines = f.readlines()
|
||||
lines = [line for line in lines if not line.strip() in ignore_list]
|
||||
for line in lines:
|
||||
try:
|
||||
fetch_lines([line])
|
||||
except KeyboardInterrupt:
|
||||
print("Interrupted by user.")
|
||||
break
|
||||
else:
|
||||
ignore_list.append(line.strip())
|
||||
with open(IGNORE_FILE, "a") as f:
|
||||
f.write(line)
|
||||
print("Done.")
|
||||
asyncio.run(fetch.process_days_concurrently(lines))
|
||||
logger.info("Fetching process completed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user