Schema Diagram and Graph Statistics
Node Statistics
| Node Label | Count |
|---|---|
| Accident | 1,798,398 |
| CrashRF | 30 |
| Weather | 14 |
| Event | 1,282,627 |
| Vehicle | 2,707,404 |
| Damage | 26 |
| Distract | 22 |
| DrImpair | 14 |
| DriverRF | 80 |
| Factor | 24 |
| Maneuver | 10 |
| Trailer | 9,665 |
| VehicleSF | 37 |
| Violatn | 77 |
| Vision | 19 |
| Person | 4,673,559 |
| Drug | 816 |
| DrugEvent | 1,475,978 |
| DrugTest | 12 |
| NMDistract | 19 |
| NMCrash | 24 |
| NMImpair | 13 |
| NMPrior | 17 |
| PersonRF | 81 |
| Race | 20 |
| SafetyEquipment | 6 |
Edge Statistics
| Relationship | Count |
|---|---|
| COMMITTED_BY | 503,258 |
| CRASH_FACTOR_OF | 75,484 |
| DISTRACTION_OF | 324,970 |
| DRIVER_RF_OF | 2,323,085 |
| DRUG_EVENT_OF | 1,475,978 |
| DRUG_TEST_OF | 1,891,459 |
| FACTOR_OF | 195,958 |
| HAS_OCCUPANT | 4,304,217 |
| HAS_SAFETY_EQUIPMENT | 4,425 |
| HAS_TRAILER | 9,665 |
| HAS_VICTIM | 4,673,559 |
| IMPAIRMENT_OF | 433,813 |
| INVOLVES | 2,707,404 |
| MANEUVER_OF | 682,710 |
| NMCRASH_OF | 247,805 |
| NMDISTRACT_OF | 29,550 |
| NMIMPAIR_OF | 70,181 |
| NMPRIOR_OF | 99,042 |
| OBJECT_OF | 240,611 |
| OCCURRED_IN | 1,913,545 |
| PERSON_RF_OF | 90,186 |
| POSITION_OF | 4,691,688 |
| RACE_OF | 928,011 |
| RESULT_OF | 1,451,749 |
| RESULTED_IN | 1,282,627 |
| SUBJECT_OF | 1,282,627 |
| VEHICLE_SF_OF | 195,314 |
| VISION_OF | 112,838 |
Disk space: 18.66 GiB
Total import time: 13,567.84 Seconds
Data Loading
Select a button below to view its how to import this data to the database.
Under the subtitles for some specific node/edge labels, we included average import time for a specific period of time. We separate these time based on years, since the FARS dataset evolved over time. If there has been a significant change to the contents and structure of a certain FARS table, it will be reflected in the import time accordingly.
Core Entities
Additional Entities
Relationships
Accident Nodes (1038.627s)
Average import time per year: 21.638s across 1975-2022.
python
#!/usr/bin/env python3
"""
Import all FARS accident.csv files into Neo4j as :Accident nodes.
This reads the normalized raw files at:
<base>/<year>/National/accident.csv
It builds a master property list from the union of every ACCIDENT column across
1975-2022. If a year does not have a column, the importer writes a blank string
for that property so all :Accident nodes share the same property surface.
The Neo4j table id is accident_id = <year>_<ST_CASE>. The original FARS
ST_CASE column is kept unchanged.
"""
from __future__ import annotations
import argparse
import math
import os
import re
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 5000
YEAR_MIN = 1975
YEAR_MAX = 2022
MERGE_QUERY = """
UNWIND $rows AS row
MERGE (a:Accident {accident_id: row.accident_id})
SET a += row
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Import all FARS accident tables into Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument(
"--create-database",
action="store_true",
help="Create the target Neo4j database from the system database if it does not exist.",
)
parser.add_argument(
"--ensure-constraint-only",
action="store_true",
help="Only ensure the :Accident(accident_id) unique constraint, then exit without importing rows.",
)
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument(
"--missing-mode",
choices=("empty", "null"),
default=os.getenv("FARS_MISSING_MODE", "empty"),
help=(
"Use 'empty' to store blank strings for missing columns. "
"Use 'null' to omit missing properties in Neo4j."
),
)
parser.add_argument("--dry-run", action="store_true", help="Inspect files/columns without connecting to Neo4j.")
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def normalize_column(name: Any) -> str:
return (
str(name)
.replace("\ufeff", "")
.replace("", "")
.strip()
.upper()
)
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def find_accident_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if YEAR_MIN <= year <= YEAR_MAX:
accident_path = year_dir / "National" / "accident.csv"
if accident_path.exists():
files.append((year, accident_path))
else:
print(f"WARNING: missing {accident_path}")
return files
def read_header(csv_path: Path) -> list[str]:
df = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
return normalize_columns(df.columns)
def build_master_columns(files: list[tuple[int, Path]]) -> tuple[list[str], dict[str, set[int]]]:
master_columns: list[str] = []
seen: set[str] = set()
presence: dict[str, set[int]] = {}
for year, csv_path in files:
columns = read_header(csv_path)
if "ST_CASE" not in columns:
raise ValueError(f"{csv_path} does not contain ST_CASE")
for column in columns:
presence.setdefault(column, set()).add(year)
if column not in seen:
seen.add(column)
master_columns.append(column)
for extra in ("accident_id", "DATA_YEAR"):
if extra not in seen:
master_columns.append(extra)
presence[extra] = {year for year, _ in files}
return master_columns, presence
def count_rows(csv_path: Path) -> int:
with csv_path.open("r", encoding="latin-1", errors="replace", newline="") as fh:
return max(sum(1 for _ in fh) - 1, 0)
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def clean_value(value: Any, missing_value: Any) -> Any:
if value is None:
return missing_value
try:
if pd.isna(value):
return missing_value
except TypeError:
pass
if hasattr(value, "item") and not isinstance(value, str):
value = value.item()
if isinstance(value, str):
text = value.strip()
return text if text else missing_value
if isinstance(value, float):
if math.isnan(value):
return missing_value
if math.isfinite(value) and value.is_integer():
return int(value)
return float(value)
return value
def clean_row(row: dict[str, Any], missing_value: Any) -> dict[str, Any]:
return {key: clean_value(value, missing_value) for key, value in row.items()}
def add_import_columns(chunk: pd.DataFrame, year: int) -> pd.DataFrame:
st_case = chunk["ST_CASE"].map(identifier_part)
chunk["accident_id"] = st_case.map(lambda value: f"{year}_{value}")
chunk["DATA_YEAR"] = year
return chunk
def iter_clean_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=batch_size,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_import_columns(chunk, year)
for column in master_columns:
if column not in chunk.columns:
chunk[column] = missing_value
chunk = chunk[master_columns]
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [
clean_row(row, missing_value)
for row in chunk.to_dict(orient="records")
]
def import_batch(tx: Any, rows: list[dict[str, Any]]) -> None:
tx.run(MERGE_QUERY, rows=rows).consume()
def write_batch(session: Any, rows: list[dict[str, Any]]) -> None:
if hasattr(session, "execute_write"):
session.execute_write(import_batch, rows)
else:
session.write_transaction(import_batch, rows)
def quote_database_name(database: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.-]+", database):
raise ValueError(f"Unsafe Neo4j database name: {database!r}")
return f"`{database}`"
def get_databases(driver: Any) -> dict[str, str]:
with driver.session(database="system") as session:
result = session.run("SHOW DATABASES YIELD name, currentStatus RETURN name, currentStatus")
return {record["name"]: record["currentStatus"] for record in result}
def wait_for_database(driver: Any, database: str, timeout_seconds: int = 120) -> None:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
databases = get_databases(driver)
status = databases.get(database)
if status == "online":
return
time.sleep(2)
status = get_databases(driver).get(database)
raise TimeoutError(f"Database {database!r} did not become online; last status was {status!r}")
def ensure_database(driver: Any, database: str, create_database: bool) -> None:
databases = get_databases(driver)
status = databases.get(database)
if status == "online":
return
if status is not None:
print(f"Database {database!r} exists with status {status!r}; waiting for online...")
wait_for_database(driver, database)
return
if not create_database:
raise SystemExit(
f"Neo4j database {database!r} does not exist. "
f"Re-run with --create-database if you want this script to create it."
)
quoted_database = quote_database_name(database)
print(f"Creating Neo4j database {database!r}...")
with driver.session(database="system") as session:
try:
session.run(f"CREATE DATABASE {quoted_database} IF NOT EXISTS").consume()
except Exception as exc:
raise SystemExit(
f"Could not create Neo4j database {database!r}. "
"The server or current user does not support CREATE DATABASE. "
"Create the database manually, or run this importer against an existing database."
) from exc
wait_for_database(driver, database)
def ensure_constraint(session: Any) -> None:
queries = [
"CREATE CONSTRAINT accident_accident_id IF NOT EXISTS FOR (a:Accident) REQUIRE a.accident_id IS UNIQUE",
"CREATE CONSTRAINT ON (a:Accident) ASSERT a.accident_id IS UNIQUE",
]
last_error: Exception | None = None
for query in queries:
try:
session.run(query).consume()
return
except Exception as exc: # Neo4j syntax differs by version.
last_error = exc
print(f"WARNING: could not create unique constraint on :Accident(accident_id): {last_error}")
def print_dry_run(files: list[tuple[int, Path]], master_columns: list[str], presence: dict[str, set[int]]) -> None:
years = [year for year, _ in files]
missing_some_years = [
column
for column, column_years in presence.items()
if len(column_years) != len(files)
]
total_rows = sum(count_rows(path) for _, path in files)
print(f"Accident files: {len(files)} ({min(years)}-{max(years)})")
print(f"Accident rows : {total_rows:,}")
print(f"Properties : {len(master_columns)}")
print(f"Vary by year : {len(missing_some_years)} columns")
print(f"First columns : {', '.join(master_columns[:12])}")
print(f"Last columns : {', '.join(master_columns[-12:])}")
if missing_some_years:
examples = ", ".join(missing_some_years[:20])
print(f"Examples with year gaps: {examples}")
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
missing_value = "" if args.missing_mode == "empty" else None
start = time.time()
if args.ensure_constraint_only:
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
ensure_database(driver, args.database, args.create_database)
with driver.session(database=args.database) as session:
ensure_constraint(session)
print(f"Constraint ensured on database {args.database!r}: :Accident(accident_id)")
finally:
driver.close()
return
files = find_accident_files(args.base)
if not files:
raise SystemExit(f"No accident.csv files found under {args.base}")
master_columns, presence = build_master_columns(files)
if args.dry_run:
print_dry_run(files, master_columns, presence)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
driver.verify_connectivity()
ensure_database(driver, args.database, args.create_database)
total_nodes = 0
print(f"Accident files : {len(files)}")
print(f"Properties : {len(master_columns)}")
print(f"Missing values : {repr(missing_value)}")
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Node load started UTC : {utc_now()}")
try:
with driver.session(database=args.database) as session:
ensure_constraint(session)
for index, (year, csv_path) in enumerate(files, 1):
year_start = time.time()
year_nodes = 0
try:
for rows in iter_clean_batches(csv_path, year, master_columns, args.batch_size, missing_value):
if not rows:
continue
write_batch(session, rows)
year_nodes += len(rows)
total_nodes += len(rows)
elapsed = time.time() - year_start
print(
f"[{index}/{len(files)}] Done {year}: "
f"{year_nodes:,} nodes in {seconds_text(elapsed)} "
f"({format_duration(elapsed)})"
)
except Exception as exc:
elapsed = time.time() - year_start
print(
f"[{index}/{len(files)}] FAILED {year} after "
f"{seconds_text(elapsed)} ({format_duration(elapsed)}): {exc}"
)
finally:
driver.close()
load_elapsed = time.time() - load_start
total_elapsed = time.time() - start
print(f"Node load finished UTC : {utc_now()}")
print(f"Node load seconds : {load_elapsed:.3f}")
print(f"Total script seconds : {total_elapsed:.3f}")
print(f"\nTotal Accident nodes: {total_nodes:,} in {seconds_text(load_elapsed)} ({format_duration(load_elapsed)})")
if __name__ == "__main__":
main()
Vehicle Nodes (4050.073s)
Average combined import time per year: 84.376s across 1975-2022.
python
#!/usr/bin/env python3
"""
Import all FARS vehicle-family CSV files into Neo4j as :Vehicle nodes.
This reads the normalized raw files at:
<base>/<year>/National/vehicle.csv
<base>/<year>/National/vehnit.csv
<base>/<year>/National/parkwork.csv
It builds a master property list from the union of every source column across
VEHICLE, VEHNIT, and PARKWORK. PARKWORK's parked/working vehicle columns are
canonicalized to their parallel VEHICLE names, e.g. PMAKE -> MAKE and
PVEH_SEV -> DEFORMED. If a source file does not have a column, the importer
writes a blank string for that property so all :Vehicle nodes share the same
property surface.
The Neo4j table id is vehicle_id = <year>_<ST_CASE>_<VEH_NO>. The original FARS
ST_CASE and VEH_NO columns are kept unchanged. accident_id = <year>_<ST_CASE>
matches the :Accident(accident_id) key.
source_table preserves the physical source table. transport_status and
vehicle_source_class provide easier graph query predicates:
VEHICLE -> in_transport
VEHNIT -> not_in_transport / parked_working
PARKWORK -> not_in_transport / parked_working
"""
from __future__ import annotations
import argparse
import math
import os
import re
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 5000
YEAR_MIN = 1975
YEAR_MAX = 2022
TABLES = ("vehicle", "vehnit", "parkwork")
# PARKWORK is the 2010+ replacement/parallel table for not-in-transport
# parked/working vehicles. Most columns are a simple leading-P form of the
# VEHICLE column. These irregular names are documented by the FARS manual.
PARKWORK_COLUMN_MAP = {
"PTYPE": "UNITTYPE",
"PTYPENAME": "UNITTYPENAME",
"PBODYTYP": "BODY_TYP",
"PBODYTYPNAME": "BODY_TYPNAME",
"PMODYEAR": "MOD_YEAR",
"PMODYEARNAME": "MOD_YEARNAME",
"PFUECODE": "FUELCODE",
"PTRAILER": "TOW_VEH",
"PTRAILERNAME": "TOW_VEHNAME",
"PCARGTYP": "CARGO_BT",
"PCARGTYPNAME": "CARGO_BTNAME",
"PHAZPLAC": "HAZ_PLAC",
"PHAZPLACNAME": "HAZ_PLACNAME",
"PSP_USE": "SPEC_USE",
"PSP_USENAME": "SPEC_USENAME",
"PEM_USE": "EMER_USE",
"PEM_USENAME": "EMER_USENAME",
"PVEH_SEV": "DEFORMED",
"PVEH_SEVNAME": "DEFORMEDNAME",
"PFIRE": "FIRE_EXP",
"PFIRENAME": "FIRE_EXPNAME",
}
MERGE_QUERY = """
UNWIND $rows AS row
MERGE (v:Vehicle {vehicle_id: row.vehicle_id})
SET v += row
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Import all FARS vehicle-family tables into Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument(
"--create-database",
action="store_true",
help="Create the target Neo4j database from the system database if it does not exist.",
)
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Vehicle constraints/indexes, then exit without importing rows.",
)
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument(
"--missing-mode",
choices=("empty", "null"),
default=os.getenv("FARS_MISSING_MODE", "empty"),
help=(
"Use 'empty' to store blank strings for missing columns. "
"Use 'null' to omit missing properties in Neo4j."
),
)
parser.add_argument("--check-ids", action="store_true", help="During dry-run, verify vehicle_id uniqueness.")
parser.add_argument("--dry-run", action="store_true", help="Inspect files/columns without connecting to Neo4j.")
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def normalize_column(name: Any) -> str:
return (
str(name)
.replace("\ufeff", "")
.replace("", "")
.strip()
.upper()
)
def canonicalize_column(name: Any, table: str | None = None) -> str:
column = normalize_column(name)
if table == "PARKWORK":
if column in PARKWORK_COLUMN_MAP:
return PARKWORK_COLUMN_MAP[column]
if column not in {"STATE", "STATENAME", "ST_CASE", "VEH_NO"} and column.startswith("P"):
return column[1:]
return column
def normalize_columns(columns: Iterable[Any], table: str | None = None) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = canonicalize_column(column, table)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def find_vehicle_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if YEAR_MIN <= year <= YEAR_MAX:
national_dir = year_dir / "National"
for table in TABLES:
csv_path = national_dir / f"{table}.csv"
if csv_path.exists():
files.append((year, table.upper(), csv_path))
return files
def read_header(csv_path: Path, table: str) -> list[str]:
df = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
return normalize_columns(df.columns, table)
def build_master_columns(files: list[tuple[int, str, Path]]) -> tuple[list[str], dict[str, set[str]]]:
master_columns: list[str] = []
seen: set[str] = set()
presence: dict[str, set[str]] = {}
for year, table, csv_path in files:
columns = read_header(csv_path, table)
missing_keys = {"ST_CASE", "VEH_NO"} - set(columns)
if missing_keys:
raise ValueError(f"{csv_path} is missing required columns: {sorted(missing_keys)}")
source_key = f"{year}:{table}"
for column in columns:
presence.setdefault(column, set()).add(source_key)
if column not in seen:
seen.add(column)
master_columns.append(column)
for extra in (
"vehicle_id",
"accident_id",
"DATA_YEAR",
"source_table",
"transport_status",
"vehicle_source_class",
):
if extra not in seen:
master_columns.append(extra)
presence[extra] = {f"{year}:{table}" for year, table, _ in files}
return master_columns, presence
def count_rows(csv_path: Path) -> int:
with csv_path.open("r", encoding="latin-1", errors="replace", newline="") as fh:
return max(sum(1 for _ in fh) - 1, 0)
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def clean_value(value: Any, missing_value: Any) -> Any:
if value is None:
return missing_value
try:
if pd.isna(value):
return missing_value
except TypeError:
pass
if hasattr(value, "item") and not isinstance(value, str):
value = value.item()
if isinstance(value, str):
text = value.strip()
return text if text else missing_value
if isinstance(value, float):
if math.isnan(value):
return missing_value
if math.isfinite(value) and value.is_integer():
return int(value)
return float(value)
return value
def clean_row(row: dict[str, Any], missing_value: Any) -> dict[str, Any]:
return {key: clean_value(value, missing_value) for key, value in row.items()}
def transport_status_for(table: str) -> str:
return "in_transport" if table == "VEHICLE" else "not_in_transport"
def vehicle_source_class_for(table: str) -> str:
return "in_transport" if table == "VEHICLE" else "parked_working"
def add_import_columns(chunk: pd.DataFrame, year: int, table: str) -> pd.DataFrame:
st_case = chunk["ST_CASE"].map(identifier_part)
veh_no = chunk["VEH_NO"].map(identifier_part)
chunk["accident_id"] = st_case.map(lambda value: f"{year}_{value}")
chunk["vehicle_id"] = [
f"{accident_id}_{vehicle_number}"
for accident_id, vehicle_number in zip(chunk["accident_id"], veh_no)
]
chunk["DATA_YEAR"] = year
chunk["source_table"] = table
chunk["transport_status"] = transport_status_for(table)
chunk["vehicle_source_class"] = vehicle_source_class_for(table)
return chunk
def iter_clean_batches(
csv_path: Path,
year: int,
table: str,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=batch_size,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns, table)
chunk = add_import_columns(chunk, year, table)
chunk = chunk.reindex(columns=master_columns, fill_value=missing_value)
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [
clean_row(row, missing_value)
for row in chunk.to_dict(orient="records")
]
def import_batch(tx: Any, rows: list[dict[str, Any]]) -> None:
tx.run(MERGE_QUERY, rows=rows).consume()
def write_batch(session: Any, rows: list[dict[str, Any]]) -> None:
if hasattr(session, "execute_write"):
session.execute_write(import_batch, rows)
else:
session.write_transaction(import_batch, rows)
def quote_database_name(database: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.-]+", database):
raise ValueError(f"Unsafe Neo4j database name: {database!r}")
return f"`{database}`"
def get_databases(driver: Any) -> dict[str, str]:
with driver.session(database="system") as session:
result = session.run("SHOW DATABASES YIELD name, currentStatus RETURN name, currentStatus")
return {record["name"]: record["currentStatus"] for record in result}
def wait_for_database(driver: Any, database: str, timeout_seconds: int = 120) -> None:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
databases = get_databases(driver)
status = databases.get(database)
if status == "online":
return
time.sleep(2)
status = get_databases(driver).get(database)
raise TimeoutError(f"Database {database!r} did not become online; last status was {status!r}")
def ensure_database(driver: Any, database: str, create_database: bool) -> None:
databases = get_databases(driver)
status = databases.get(database)
if status == "online":
return
if status is not None:
print(f"Database {database!r} exists with status {status!r}; waiting for online...")
wait_for_database(driver, database)
return
if not create_database:
raise SystemExit(
f"Neo4j database {database!r} does not exist. "
f"Re-run with --create-database if you want this script to create it."
)
quoted_database = quote_database_name(database)
print(f"Creating Neo4j database {database!r}...")
with driver.session(database="system") as session:
try:
session.run(f"CREATE DATABASE {quoted_database} IF NOT EXISTS").consume()
except Exception as exc:
raise SystemExit(
f"Could not create Neo4j database {database!r}. "
"The server or current user does not support CREATE DATABASE. "
"Create the database manually, or run this importer against an existing database."
) from exc
wait_for_database(driver, database)
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
run_ddl(
session,
"CREATE INDEX vehicle_accident_id IF NOT EXISTS FOR (v:Vehicle) ON (v.accident_id)",
":Vehicle(accident_id) index",
)
run_ddl(
session,
"CREATE INDEX vehicle_source_table IF NOT EXISTS FOR (v:Vehicle) ON (v.source_table)",
":Vehicle(source_table) index",
)
run_ddl(
session,
"CREATE INDEX vehicle_transport_status IF NOT EXISTS FOR (v:Vehicle) ON (v.transport_status)",
":Vehicle(transport_status) index",
)
def check_vehicle_ids(files: list[tuple[int, str, Path]]) -> tuple[int, int]:
seen: set[str] = set()
duplicates = 0
for year, table, csv_path in files:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=100_000,
usecols=lambda c: normalize_column(c) in {"ST_CASE", "VEH_NO"},
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns, table)
st_case = chunk["ST_CASE"].map(identifier_part)
veh_no = chunk["VEH_NO"].map(identifier_part)
for accident_case, vehicle_number in zip(st_case, veh_no):
vehicle_id = f"{year}_{accident_case}_{vehicle_number}"
if vehicle_id in seen:
duplicates += 1
else:
seen.add(vehicle_id)
return len(seen), duplicates
def print_dry_run(
files: list[tuple[int, str, Path]],
master_columns: list[str],
presence: dict[str, set[str]],
check_ids: bool,
) -> None:
table_counts = Counter(table for _, table, _ in files)
row_counts = Counter()
total_rows = 0
for _, table, csv_path in files:
rows = count_rows(csv_path)
row_counts[table] += rows
total_rows += rows
missing_some_sources = [
column
for column, source_files in presence.items()
if len(source_files) != len(files)
]
years = [year for year, _, _ in files]
print(f"Vehicle-family files: {len(files)} ({min(years)}-{max(years)})")
for table in TABLES:
label = table.upper()
print(f" {label:<8} files: {table_counts[label]:>2} rows: {row_counts[label]:,}")
print(f"Total rows : {total_rows:,}")
print(f"Properties : {len(master_columns)}")
print(f"Vary by source/year : {len(missing_some_sources)} columns")
print(f"First columns : {', '.join(master_columns[:12])}")
print(f"Last columns : {', '.join(master_columns[-12:])}")
if missing_some_sources:
examples = ", ".join(missing_some_sources[:20])
print(f"Examples with gaps : {examples}")
if check_ids:
unique_ids, duplicates = check_vehicle_ids(files)
print(f"Unique vehicle_id : {unique_ids:,}")
print(f"Duplicate vehicle_id: {duplicates:,}")
def import_files(
session: Any,
files: list[tuple[int, str, Path]],
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> int:
total_rows = 0
for index, (year, table, csv_path) in enumerate(files, 1):
file_start = time.time()
file_rows = 0
try:
for rows in iter_clean_batches(csv_path, year, table, master_columns, batch_size, missing_value):
if not rows:
continue
write_batch(session, rows)
file_rows += len(rows)
total_rows += len(rows)
elapsed = time.time() - file_start
print(
f"[{index}/{len(files)}] Done {year} {table}: "
f"{file_rows:,} rows in {seconds_text(elapsed)} ({format_duration(elapsed)})"
)
except Exception as exc:
elapsed = time.time() - file_start
print(
f"[{index}/{len(files)}] FAILED {year} {table} after "
f"{seconds_text(elapsed)} ({format_duration(elapsed)}): {exc}"
)
return total_rows
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
missing_value = "" if args.missing_mode == "empty" else None
start = time.time()
files = find_vehicle_files(args.base)
if not files:
raise SystemExit(f"No vehicle-family CSV files found under {args.base}")
master_columns, presence = build_master_columns(files)
if args.dry_run:
print_dry_run(files, master_columns, presence, args.check_ids)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
ensure_database(driver, args.database, args.create_database)
if args.ensure_constraints_only:
with driver.session(database=args.database) as session:
ensure_constraints(session)
print(f"Constraints/indexes ensured on database {args.database!r}")
return
print(f"Vehicle-family files : {len(files)}")
print(f"Properties : {len(master_columns)}")
print(f"Missing values : {repr(missing_value)}")
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Node load started UTC: {utc_now()}")
with driver.session(database=args.database) as session:
ensure_constraints(session)
total_rows = import_files(session, files, master_columns, args.batch_size, missing_value)
load_elapsed = time.time() - load_start
total_elapsed = time.time() - start
print(f"Node load finished UTC: {utc_now()}")
print(f"Node load seconds : {load_elapsed:.3f}")
print(f"Total script seconds : {total_elapsed:.3f}")
print(f"\nTotal Vehicle rows processed: {total_rows:,} in {seconds_text(load_elapsed)} ({format_duration(load_elapsed)})")
finally:
driver.close()
if __name__ == "__main__":
main()
Person Nodes (4771.704s)
Average combined import time per year: 99.398s across 1975-2022.
python
#!/usr/bin/env python3
"""
Import FARS PERSON data and person-level supplements into Neo4j.
Model:
(:Person {person_id: <year>_<ST_CASE>_<VEH_NO>_<PER_NO>})
PERSON rows are imported as Person nodes with every source column preserved.
PBTYPE is one-to-one with Person, so PBTYPE columns are appended to Person with
a PBTYPE_ prefix to avoid overwriting PERSON columns such as AGE or SEX.
SAFETYEQ is mapped onto a small controlled set of SafetyEquipment category
nodes. From 2010-2016, MSAFEQMT rows are mapped by code. From 2017 onward, NM*
flag columns are mapped by their "Yes" values:
(:Person)-[:HAS_SAFETY_EQUIPMENT]->(:SafetyEquipment)
The 2010-2016 code 7 means "other safety equipment"; because that combines
"other protective" and "other preventive" concepts, the importer creates both
edges and sets or_flag=1 only on those two relationships.
Run this after Accident and Vehicle nodes are present if you want the Person
nodes to carry accident_id and vehicle_id keys that can be linked later. The
script itself only creates Person and SafetyEquipment nodes plus the direct
Person-to-SafetyEquipment relationship.
"""
from __future__ import annotations
import argparse
import math
import os
import re
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 5000
PERSON_TABLE = "person"
PBTYPE_TABLE = "pbtype"
SAFETYEQ_TABLE = "safetyeq"
SAFETY_EQUIPMENT_TYPES: dict[str, str] = {
"helmet": "Helmet",
"protective_pads": "Protective Pads",
"other_protective": "Other Protective Equipment",
"reflective_clothing": "Reflective Clothing",
"lighting": "Lighting",
"other_preventive": "Other Preventive Equipment",
}
OLD_SAFETYEQ_CODE_MAP: dict[str, list[tuple[str, int | None]]] = {
"2": [("helmet", None)],
"3": [("reflective_clothing", None)],
"4": [("protective_pads", None)],
"5": [("lighting", None)],
"7": [("other_protective", 1), ("other_preventive", 1)],
}
NEW_SAFETYEQ_FLAG_MAP: dict[str, tuple[str, str]] = {
"NMHELMET": ("helmet", "NMHELMETNAME"),
"NMPROPAD": ("protective_pads", "NMPROPADNAME"),
"NMOTHPRO": ("other_protective", "NMOTHPRONAME"),
"NMREFCLO": ("reflective_clothing", "NMREFCLONAME"),
"NMLIGHT": ("lighting", "NMLIGHTNAME"),
"NMOTHPRE": ("other_preventive", "NMOTHPRENAME"),
}
MERGE_PERSON_QUERY = """
UNWIND $rows AS row
MERGE (p:Person {person_id: row.person_id})
SET p += row
RETURN count(p) AS matched
"""
UPDATE_PBTYPE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
SET p += row
RETURN count(p) AS matched
"""
IMPORT_SAFETYEQ_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MERGE (s:SafetyEquipment {safety_equipment_id: row.safety_equipment_id})
SET s.name = row.safety_equipment_name,
s.source_table = "SAFETYEQ"
MERGE (p)-[r:HAS_SAFETY_EQUIPMENT]->(s)
SET p.safetyeq_present = true,
r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_column = row.source_column,
r.source_value = row.source_value,
r.source_name = row.source_name
FOREACH (_ IN CASE WHEN row.or_flag = 1 THEN [1] ELSE [] END | SET r.or_flag = 1)
FOREACH (_ IN CASE WHEN row.or_flag = 1 THEN [] ELSE [1] END | REMOVE r.or_flag)
RETURN count(r) AS matched
"""
UPDATE_SAFETYEQ_COUNTS_QUERY = """
MATCH (p:Person)-[:HAS_SAFETY_EQUIPMENT]->(s:SafetyEquipment)
WITH p, count(s) AS safetyeq_count
SET p.safetyeq_count = safetyeq_count
RETURN count(p) AS persons_with_safetyeq
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Import FARS Person, PBTYPE, and SAFETYEQ data into Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument(
"--missing-mode",
choices=("empty", "null"),
default=os.getenv("FARS_MISSING_MODE", "empty"),
help=(
"Use 'empty' to store blank strings for missing columns. "
"Use 'null' to omit missing properties in Neo4j."
),
)
parser.add_argument("--skip-person", action="store_true")
parser.add_argument("--skip-pbtype", action="store_true")
parser.add_argument("--skip-safetyeq", action="store_true")
parser.add_argument("--ensure-constraints-only", action="store_true")
parser.add_argument("--dry-run", action="store_true")
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return (
str(name)
.replace("\ufeff", "")
.replace("", "")
.strip()
.upper()
)
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def prefixed_pbtype_column(column: str) -> str:
return f"PBTYPE_{column}"
def find_files(base: Path, table: str) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
csv_path = year_dir / "National" / f"{table}.csv"
if csv_path.exists():
files.append((int(year_dir.name), csv_path))
return files
def read_header(csv_path: Path) -> list[str]:
df = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
return normalize_columns(df.columns)
def build_person_columns(files: list[tuple[int, Path]]) -> list[str]:
columns: list[str] = []
seen: set[str] = set()
for _, csv_path in files:
for column in read_header(csv_path):
if column not in seen:
seen.add(column)
columns.append(column)
for extra in ("person_id", "accident_id", "vehicle_id", "DATA_YEAR", "source_table"):
if extra not in seen:
seen.add(extra)
columns.append(extra)
return columns
def build_pbtype_columns(files: list[tuple[int, Path]]) -> list[str]:
columns: list[str] = []
seen: set[str] = set()
for _, csv_path in files:
for column in read_header(csv_path):
prefixed = prefixed_pbtype_column(column)
if prefixed not in seen:
seen.add(prefixed)
columns.append(prefixed)
for extra in ("person_id", "accident_id", "vehicle_id", "pbtype_year", "pbtype_present"):
if extra not in seen:
seen.add(extra)
columns.append(extra)
return columns
def build_safetyeq_relationship_columns() -> list[str]:
return [
"person_id",
"safety_equipment_id",
"safety_equipment_name",
"DATA_YEAR",
"source_table",
"source_column",
"source_value",
"source_name",
"or_flag",
]
def count_rows(csv_path: Path) -> int:
with csv_path.open("r", encoding="latin-1", errors="replace", newline="") as fh:
return max(sum(1 for _ in fh) - 1, 0)
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def clean_value(value: Any, missing_value: Any) -> Any:
if value is None:
return missing_value
try:
if pd.isna(value):
return missing_value
except (TypeError, ValueError):
pass
if hasattr(value, "item") and not isinstance(value, str):
value = value.item()
if isinstance(value, str):
text = value.strip()
return text if text else missing_value
if isinstance(value, float):
if math.isnan(value):
return missing_value
if math.isfinite(value) and value.is_integer():
return int(value)
return float(value)
return value
def clean_row(row: dict[str, Any], missing_value: Any) -> dict[str, Any]:
return {key: clean_value(value, missing_value) for key, value in row.items()}
def add_person_keys(chunk: pd.DataFrame, year: int) -> pd.DataFrame:
st_case = chunk["ST_CASE"].map(identifier_part)
veh_no = chunk["VEH_NO"].map(identifier_part)
per_no = chunk["PER_NO"].map(identifier_part)
chunk["accident_id"] = st_case.map(lambda value: f"{year}_{value}")
chunk["vehicle_id"] = [
f"{accident_id}_{vehicle_number}"
for accident_id, vehicle_number in zip(chunk["accident_id"], veh_no)
]
chunk["person_id"] = [
f"{vehicle_id}_{person_number}"
for vehicle_id, person_number in zip(chunk["vehicle_id"], per_no)
]
return chunk
def is_yes_flag(value: Any, name_value: Any = None) -> bool:
code = identifier_part(value)
if code == "2":
return True
if name_value is not None:
return str(name_value).strip().upper() == "YES"
return False
def iter_person_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(csv_path, encoding="latin-1", low_memory=False, chunksize=batch_size)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_person_keys(chunk, year)
chunk["DATA_YEAR"] = year
chunk["source_table"] = "PERSON"
chunk = chunk.reindex(columns=master_columns, fill_value=missing_value)
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [clean_row(row, missing_value) for row in chunk.to_dict(orient="records")]
def iter_pbtype_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(csv_path, encoding="latin-1", low_memory=False, chunksize=batch_size)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_person_keys(chunk, year)
chunk["pbtype_year"] = year
chunk["pbtype_present"] = True
rename = {
column: prefixed_pbtype_column(column)
for column in chunk.columns
if column not in {"person_id", "accident_id", "vehicle_id", "pbtype_year", "pbtype_present"}
}
chunk = chunk.rename(columns=rename)
chunk = chunk.reindex(columns=master_columns, fill_value=missing_value)
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [clean_row(row, missing_value) for row in chunk.to_dict(orient="records")]
def iter_safetyeq_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
del master_columns, missing_value
chunks = pd.read_csv(csv_path, encoding="latin-1", low_memory=False, chunksize=batch_size)
pending: list[dict[str, Any]] = []
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_person_keys(chunk, year)
for row in chunk.to_dict(orient="records"):
person_id = row["person_id"]
if "MSAFEQMT" in row:
source_value = identifier_part(row.get("MSAFEQMT"))
source_name = clean_value(row.get("MSAFEQMTNAME"), "")
for equipment_id, or_flag in OLD_SAFETYEQ_CODE_MAP.get(source_value, []):
pending.append(
{
"person_id": person_id,
"safety_equipment_id": equipment_id,
"safety_equipment_name": SAFETY_EQUIPMENT_TYPES[equipment_id],
"DATA_YEAR": year,
"source_table": "SAFETYEQ",
"source_column": "MSAFEQMT",
"source_value": source_value,
"source_name": source_name,
"or_flag": or_flag,
}
)
else:
for source_column, (equipment_id, name_column) in NEW_SAFETYEQ_FLAG_MAP.items():
if source_column not in row:
continue
source_name = row.get(name_column)
if not is_yes_flag(row.get(source_column), source_name):
continue
pending.append(
{
"person_id": person_id,
"safety_equipment_id": equipment_id,
"safety_equipment_name": SAFETY_EQUIPMENT_TYPES[equipment_id],
"DATA_YEAR": year,
"source_table": "SAFETYEQ",
"source_column": source_column,
"source_value": identifier_part(row.get(source_column)),
"source_name": clean_value(source_name, ""),
"or_flag": None,
}
)
if len(pending) >= batch_size:
yield [clean_row(rel, None) for rel in pending]
pending = []
if pending:
yield [clean_row(rel, None) for rel in pending]
def quote_database_name(database: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.-]+", database):
raise ValueError(f"Unsafe Neo4j database name: {database!r}")
return f"`{database}`"
def get_databases(driver: Any) -> dict[str, str]:
with driver.session(database="system") as session:
result = session.run("SHOW DATABASES YIELD name, currentStatus RETURN name, currentStatus")
return {record["name"]: record["currentStatus"] for record in result}
def ensure_database(driver: Any, database: str) -> None:
status = get_databases(driver).get(database)
if status == "online":
return
raise SystemExit(f"Neo4j database {database!r} is not online; current status is {status!r}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
run_ddl(
session,
"CREATE INDEX person_accident_id IF NOT EXISTS FOR (p:Person) ON (p.accident_id)",
":Person(accident_id) index",
)
run_ddl(
session,
"CREATE INDEX person_vehicle_id IF NOT EXISTS FOR (p:Person) ON (p.vehicle_id)",
":Person(vehicle_id) index",
)
run_ddl(
session,
"CREATE INDEX person_pbtype_present IF NOT EXISTS FOR (p:Person) ON (p.pbtype_present)",
":Person(pbtype_present) index",
)
run_ddl(
session,
"CREATE CONSTRAINT safety_equipment_id IF NOT EXISTS FOR (s:SafetyEquipment) REQUIRE s.safety_equipment_id IS UNIQUE",
":SafetyEquipment(safety_equipment_id) unique constraint",
)
def execute_batch(session: Any, query: str, rows: list[dict[str, Any]]) -> int:
result = session.run(query, rows=rows)
record = result.single()
return int(record["matched"]) if record else 0
def load_files(
session: Any,
files: list[tuple[int, Path]],
master_columns: list[str],
batch_size: int,
missing_value: Any,
label: str,
batch_iter: Any,
query: str,
) -> tuple[int, int]:
total_rows = 0
total_matched = 0
for index, (year, csv_path) in enumerate(files, 1):
start = time.time()
year_rows = 0
year_matched = 0
for rows in batch_iter(csv_path, year, master_columns, batch_size, missing_value):
matched = execute_batch(session, query, rows)
year_rows += len(rows)
year_matched += matched
total_rows += len(rows)
total_matched += matched
elapsed = time.time() - start
print(
f"[{label} {index}/{len(files)}] Done {year}: "
f"{year_matched:,}/{year_rows:,} matched in {seconds_text(elapsed)} ({format_duration(elapsed)})"
)
return total_rows, total_matched
def safetyeq_stats(files: list[tuple[int, Path]]) -> tuple[int, int, int, int, Counter[str]]:
total_rows = 0
person_ids: set[str] = set()
duplicate_rows = 0
relationship_count = 0
equipment_counts: Counter[str] = Counter()
for year, csv_path in files:
df = pd.read_csv(csv_path, encoding="latin-1", low_memory=False)
df.columns = normalize_columns(df.columns)
st_case = df["ST_CASE"].map(identifier_part)
veh_no = df["VEH_NO"].map(identifier_part)
per_no = df["PER_NO"].map(identifier_part)
person_id_values = [
f"{year}_{case_number}_{vehicle_number}_{person_number}"
for case_number, vehicle_number, person_number in zip(st_case, veh_no, per_no)
]
total_rows += len(person_id_values)
unique_this_file = set(person_id_values)
duplicate_rows += len(person_id_values) - len(unique_this_file)
person_ids.update(unique_this_file)
if "MSAFEQMT" in df.columns:
for value in df["MSAFEQMT"].map(identifier_part):
for equipment_id, _ in OLD_SAFETYEQ_CODE_MAP.get(value, []):
relationship_count += 1
equipment_counts[equipment_id] += 1
else:
for source_column, (equipment_id, name_column) in NEW_SAFETYEQ_FLAG_MAP.items():
if source_column not in df.columns:
continue
names = df[name_column] if name_column in df.columns else [None] * len(df)
for value, name_value in zip(df[source_column], names):
if is_yes_flag(value, name_value):
relationship_count += 1
equipment_counts[equipment_id] += 1
return total_rows, len(person_ids), duplicate_rows, relationship_count, equipment_counts
def print_dry_run(
person_files: list[tuple[int, Path]],
pbtype_files: list[tuple[int, Path]],
safetyeq_files: list[tuple[int, Path]],
person_columns: list[str],
pbtype_columns: list[str],
safetyeq_columns: list[str],
) -> None:
print(f"PERSON files : {len(person_files)}")
print(f"PERSON rows : {sum(count_rows(path) for _, path in person_files):,}")
print(f"PERSON properties : {len(person_columns)}")
print(f"PBTYPE files : {len(pbtype_files)}")
print(f"PBTYPE rows : {sum(count_rows(path) for _, path in pbtype_files):,}")
print(f"PBTYPE appended props : {len(pbtype_columns)}")
(
safetyeq_rows,
safetyeq_persons,
safetyeq_duplicate_rows,
safetyeq_relationships,
safetyeq_equipment_counts,
) = safetyeq_stats(safetyeq_files)
print(f"SAFETYEQ files : {len(safetyeq_files)}")
print(f"SAFETYEQ rows : {safetyeq_rows:,}")
print(f"SAFETYEQ persons : {safetyeq_persons:,}")
print(f"SAFETYEQ duplicate rows: {safetyeq_duplicate_rows:,}")
print(f"SAFETYEQ relationships : {safetyeq_relationships:,}")
print(f"SafetyEquipment nodes : {len(SAFETY_EQUIPMENT_TYPES)}")
print(f"SAFETYEQ by equipment : {dict(sorted(safetyeq_equipment_counts.items()))}")
print(f"PERSON years : {person_files[0][0] if person_files else None}-{person_files[-1][0] if person_files else None}")
print(f"PBTYPE years : {pbtype_files[0][0] if pbtype_files else None}-{pbtype_files[-1][0] if pbtype_files else None}")
print(f"SAFETYEQ years : {safetyeq_files[0][0] if safetyeq_files else None}-{safetyeq_files[-1][0] if safetyeq_files else None}")
print(f"First PERSON props : {', '.join(person_columns[:10])}")
print(f"Last PERSON props : {', '.join(person_columns[-10:])}")
print(f"First PBTYPE props : {', '.join(pbtype_columns[:10])}")
print(f"Last PBTYPE props : {', '.join(pbtype_columns[-10:])}")
print(f"SAFETYEQ rel props : {', '.join(safetyeq_columns)}")
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
missing_value = "" if args.missing_mode == "empty" else None
person_files = [] if args.skip_person else find_files(args.base, PERSON_TABLE)
pbtype_files = [] if args.skip_pbtype else find_files(args.base, PBTYPE_TABLE)
safetyeq_files = [] if args.skip_safetyeq else find_files(args.base, SAFETYEQ_TABLE)
person_columns = build_person_columns(person_files) if person_files else []
pbtype_columns = build_pbtype_columns(pbtype_files) if pbtype_files else []
safetyeq_columns = build_safetyeq_relationship_columns() if safetyeq_files else []
if args.dry_run:
print_dry_run(
person_files,
pbtype_files,
safetyeq_files,
person_columns,
pbtype_columns,
safetyeq_columns,
)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
ensure_database(driver, args.database)
with driver.session(database=args.database) as session:
ensure_constraints(session)
if args.ensure_constraints_only:
print(f"Constraints/indexes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Missing values : {repr(missing_value)}")
print(f"Script started UTC : {utc_now()}")
print("")
person_rows = person_matched = 0
pbtype_rows = pbtype_matched = 0
safetyeq_rows = safetyeq_matched = 0
if person_files:
start = time.time()
print(f"PERSON started UTC : {utc_now()}")
person_rows, person_matched = load_files(
session,
person_files,
person_columns,
args.batch_size,
missing_value,
"PERSON",
iter_person_batches,
MERGE_PERSON_QUERY,
)
print(f"PERSON seconds : {time.time() - start:.3f}")
print("")
if pbtype_files:
start = time.time()
print(f"PBTYPE started UTC : {utc_now()}")
pbtype_rows, pbtype_matched = load_files(
session,
pbtype_files,
pbtype_columns,
args.batch_size,
missing_value,
"PBTYPE",
iter_pbtype_batches,
UPDATE_PBTYPE_QUERY,
)
print(f"PBTYPE seconds : {time.time() - start:.3f}")
print("")
if safetyeq_files:
start = time.time()
print(f"SAFETYEQ started UTC : {utc_now()}")
safetyeq_rows, safetyeq_matched = load_files(
session,
safetyeq_files,
safetyeq_columns,
args.batch_size,
missing_value,
"SAFETYEQ",
iter_safetyeq_batches,
IMPORT_SAFETYEQ_QUERY,
)
result = session.run(UPDATE_SAFETYEQ_COUNTS_QUERY).single()
persons_with_safetyeq = result["persons_with_safetyeq"] if result else 0
print(f"Updated safetyeq_count on {persons_with_safetyeq:,} Person nodes")
print(f"SAFETYEQ seconds : {time.time() - start:.3f}")
total_elapsed = time.time() - total_start
print("")
print(f"Script finished UTC : {utc_now()}")
print(f"Total script seconds : {total_elapsed:.3f}")
print(f"PERSON matched : {person_matched:,}/{person_rows:,}")
print(f"PBTYPE matched : {pbtype_matched:,}/{pbtype_rows:,}")
print(f"SAFETYEQ rels matched: {safetyeq_matched:,}/{safetyeq_rows:,}")
finally:
driver.close()
if __name__ == "__main__":
main()
Event Nodes (163.549s)
Average import time per year: 12.581s across 2010-2022.
python
#!/usr/bin/env python3
"""
Import FARS CEVENT rows into Neo4j as :Event nodes.
CEVENT starts in 2010 and stores the crash-level sequence of events. The key is:
event_id = <year>_<ST_CASE>_<EVENTNUM>
The original FARS ST_CASE and EVENTNUM columns are kept as source properties.
accident_id = <year>_<ST_CASE> matches :Accident(accident_id). VNUMBER1,
VNUMBER2, AOI1, and AOI2 are preserved as properties for later relationship
creation to Vehicle nodes if we decide to model those edges.
This importer uses the union of CEVENT columns across years. Missing columns are
stored as blank strings by default so every :Event node has the same property
surface, including the 2015+ NAME columns that are absent in 2010-2014.
"""
from __future__ import annotations
import argparse
import math
import os
import re
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 5000
YEAR_MIN = 2010
YEAR_MAX = 2022
TABLE = "cevent"
REQUIRED_COLUMNS = {"ST_CASE", "EVENTNUM"}
MERGE_EVENT_QUERY = """
UNWIND $rows AS row
MERGE (e:Event {event_id: row.event_id})
SET e += row
RETURN count(e) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Import all FARS CEVENT columns into Neo4j as Event nodes.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument(
"--missing-mode",
choices=("empty", "null"),
default=os.getenv("FARS_MISSING_MODE", "empty"),
help=(
"Use 'empty' to store blank strings for missing columns. "
"Use 'null' to omit missing properties in Neo4j."
),
)
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Event constraints/indexes, then exit without importing rows.",
)
parser.add_argument(
"--check-ids",
action="store_true",
help="During dry-run, verify event_id uniqueness across CEVENT files.",
)
parser.add_argument("--dry-run", action="store_true", help="Inspect files/columns without connecting to Neo4j.")
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return (
str(name)
.replace("\ufeff", "")
.replace("", "")
.strip()
.upper()
)
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def find_cevent_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if not (YEAR_MIN <= year <= YEAR_MAX):
continue
national_dir = year_dir / "National"
if not national_dir.is_dir():
continue
candidates = sorted(
path
for path in national_dir.iterdir()
if path.is_file()
and path.suffix.lower() == ".csv"
and path.stem.lower() == TABLE
)
if candidates:
files.append((year, candidates[0]))
return files
def read_header(csv_path: Path) -> list[str]:
df = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
return normalize_columns(df.columns)
def build_master_columns(files: list[tuple[int, Path]]) -> tuple[list[str], dict[str, set[int]]]:
master_columns: list[str] = []
seen: set[str] = set()
presence: dict[str, set[int]] = {}
for year, csv_path in files:
columns = read_header(csv_path)
missing_keys = REQUIRED_COLUMNS - set(columns)
if missing_keys:
raise ValueError(f"{csv_path} is missing required columns: {sorted(missing_keys)}")
for column in columns:
presence.setdefault(column, set()).add(year)
if column not in seen:
seen.add(column)
master_columns.append(column)
for extra in ("event_id", "accident_id", "DATA_YEAR", "source_table"):
if extra not in seen:
seen.add(extra)
master_columns.append(extra)
presence[extra] = {year for year, _ in files}
return master_columns, presence
def count_rows(csv_path: Path) -> int:
with csv_path.open("r", encoding="latin-1", errors="replace", newline="") as fh:
return max(sum(1 for _ in fh) - 1, 0)
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def clean_value(value: Any, missing_value: Any) -> Any:
if value is None:
return missing_value
try:
if pd.isna(value):
return missing_value
except (TypeError, ValueError):
pass
if hasattr(value, "item") and not isinstance(value, str):
value = value.item()
if isinstance(value, str):
text = value.strip()
return text if text else missing_value
if isinstance(value, float):
if math.isnan(value):
return missing_value
if math.isfinite(value) and value.is_integer():
return int(value)
return float(value)
return value
def clean_row(row: dict[str, Any], missing_value: Any) -> dict[str, Any]:
return {key: clean_value(value, missing_value) for key, value in row.items()}
def add_event_keys(chunk: pd.DataFrame, year: int) -> pd.DataFrame:
st_case = chunk["ST_CASE"].map(identifier_part)
eventnum = chunk["EVENTNUM"].map(identifier_part)
chunk["accident_id"] = st_case.map(lambda value: f"{year}_{value}")
chunk["event_id"] = [
f"{accident_id}_{event_number}"
for accident_id, event_number in zip(chunk["accident_id"], eventnum)
]
chunk["DATA_YEAR"] = year
chunk["source_table"] = "CEVENT"
return chunk
def iter_event_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(csv_path, encoding="latin-1", low_memory=False, chunksize=batch_size)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_event_keys(chunk, year)
chunk = chunk.reindex(columns=master_columns, fill_value=missing_value)
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [clean_row(row, missing_value) for row in chunk.to_dict(orient="records")]
def quote_database_name(database: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.-]+", database):
raise ValueError(f"Unsafe Neo4j database name: {database!r}")
return f"`{database}`"
def get_databases(driver: Any) -> dict[str, str]:
with driver.session(database="system") as session:
result = session.run("SHOW DATABASES YIELD name, currentStatus RETURN name, currentStatus")
return {record["name"]: record["currentStatus"] for record in result}
def ensure_database(driver: Any, database: str) -> None:
status = get_databases(driver).get(database)
if status == "online":
return
raise SystemExit(f"Neo4j database {database!r} is not online; current status is {status!r}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT event_id IF NOT EXISTS FOR (e:Event) REQUIRE e.event_id IS UNIQUE",
":Event(event_id) unique constraint",
)
run_ddl(
session,
"CREATE INDEX event_accident_id IF NOT EXISTS FOR (e:Event) ON (e.accident_id)",
":Event(accident_id) index",
)
run_ddl(
session,
"CREATE INDEX event_source_table IF NOT EXISTS FOR (e:Event) ON (e.source_table)",
":Event(source_table) index",
)
def execute_batch(session: Any, rows: list[dict[str, Any]]) -> int:
result = session.run(MERGE_EVENT_QUERY, rows=rows)
record = result.single()
return int(record["matched"]) if record else 0
def load_files(
session: Any,
files: list[tuple[int, Path]],
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> tuple[int, int]:
total_rows = 0
total_matched = 0
for index, (year, csv_path) in enumerate(files, 1):
start = time.time()
year_rows = 0
year_matched = 0
for rows in iter_event_batches(csv_path, year, master_columns, batch_size, missing_value):
matched = execute_batch(session, rows)
year_rows += len(rows)
year_matched += matched
total_rows += len(rows)
total_matched += matched
elapsed = time.time() - start
print(
f"[CEVENT {index}/{len(files)}] Done {year}: "
f"{year_matched:,}/{year_rows:,} matched in {seconds_text(elapsed)} ({format_duration(elapsed)})"
)
return total_rows, total_matched
def check_event_ids(files: list[tuple[int, Path]]) -> tuple[int, int]:
seen: set[str] = set()
duplicates = 0
for year, csv_path in files:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=100_000,
usecols=lambda c: normalize_column(c) in REQUIRED_COLUMNS,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
st_case = chunk["ST_CASE"].map(identifier_part)
eventnum = chunk["EVENTNUM"].map(identifier_part)
for case_number, event_number in zip(st_case, eventnum):
event_id = f"{year}_{case_number}_{event_number}"
if event_id in seen:
duplicates += 1
else:
seen.add(event_id)
return len(seen), duplicates
def print_dry_run(
files: list[tuple[int, Path]],
master_columns: list[str],
presence: dict[str, set[int]],
check_ids: bool,
) -> None:
row_counts = Counter()
total_rows = 0
for year, csv_path in files:
rows = count_rows(csv_path)
row_counts[year] = rows
total_rows += rows
missing_some_years = [
column
for column, years in presence.items()
if len(years) != len(files)
]
years = [year for year, _ in files]
print(f"CEVENT files : {len(files)}")
print(f"CEVENT years : {min(years) if years else None}-{max(years) if years else None}")
print(f"CEVENT rows : {total_rows:,}")
print(f"Event properties : {len(master_columns)}")
print(f"Vary by year : {len(missing_some_years)} columns")
print(f"First columns : {', '.join(master_columns[:12])}")
print(f"Last columns : {', '.join(master_columns[-12:])}")
if missing_some_years:
examples = ", ".join(missing_some_years[:20])
print(f"Examples with gaps : {examples}")
print("Rows by year :")
for year in sorted(row_counts):
print(f" {year}: {row_counts[year]:,}")
if check_ids:
unique_ids, duplicates = check_event_ids(files)
print(f"Unique event_id : {unique_ids:,}")
print(f"Duplicate event_id : {duplicates:,}")
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
missing_value = "" if args.missing_mode == "empty" else None
start = time.time()
files = find_cevent_files(args.base)
if not files:
raise SystemExit(f"No CEVENT CSV files found under {args.base}")
master_columns, presence = build_master_columns(files)
if args.dry_run:
print_dry_run(files, master_columns, presence, args.check_ids)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
ensure_database(driver, args.database)
with driver.session(database=args.database) as session:
ensure_constraints(session)
if args.ensure_constraints_only:
print(f"Constraints/indexes ensured on database {args.database!r}")
return
print(f"CEVENT files : {len(files)}")
print(f"Event properties : {len(master_columns)}")
print(f"Missing values : {repr(missing_value)}")
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Event load started UTC: {utc_now()}")
rows_processed, rows_matched = load_files(
session,
files,
master_columns,
args.batch_size,
missing_value,
)
load_elapsed = time.time() - load_start
total_elapsed = time.time() - start
print(f"Event load finished UTC: {utc_now()}")
print(f"Event load seconds : {load_elapsed:.3f}")
print(f"Total script seconds : {total_elapsed:.3f}")
print(
f"\nTotal Event rows processed: {rows_processed:,}; "
f"matched/merged: {rows_matched:,} in {seconds_text(load_elapsed)} ({format_duration(load_elapsed)})"
)
finally:
driver.close()
if __name__ == "__main__":
main()
Append VPIC to Vehicle Nodes (1200.171s)
Average VPICDECODE time per year: 70.263s across 2006-2022.
Average VPICTRAILERDECODE time per year: 0.656s across 2016-2022.
python
#!/usr/bin/env python3
"""
Append VPIC decode data to existing FARS :Vehicle nodes and create trailer nodes.
VPICDECODE is one row per decoded vehicle, so its columns are appended directly
to :Vehicle with a VPICDECODE_ prefix to avoid overwriting core FARS columns
such as MAKE, MODEL, BODY_TYP, or MOD_YEAR.
VPICTRAILERDECODE is one row per decoded trailer attached to a towing vehicle.
It is keyed by DATA_YEAR, ST_CASE, VEH_NO, and TRAILER_NO, where VEH_NO is the
towing vehicle and TRAILER_NO is the trailer slot. These rows are imported as
:Trailer nodes with:
trailer_id = <year>_<ST_CASE>_<VEH_NO>_<TRAILER_NO>
and connected to their towing vehicle:
(:Vehicle)-[:HAS_TRAILER]->(:Trailer)
Run this after the base Vehicle import has completed. The script uses MATCH on
Vehicle(vehicle_id), so it does not create incomplete Vehicle nodes.
"""
from __future__ import annotations
import argparse
import math
import os
import re
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 5000
VPICDECODE_TABLE = "vpicdecode"
VPICTRAILERDECODE_TABLE = "vpictrailerdecode"
UPDATE_VPICDECODE_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
SET v += row
RETURN count(v) AS matched
"""
IMPORT_TRAILER_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
MERGE (t:Trailer {trailer_id: row.trailer_id})
SET t += row
MERGE (v)-[r:HAS_TRAILER]->(t)
SET r.trailer_no = row.TRAILER_NO,
r.source_table = row.source_table,
r.DATA_YEAR = row.DATA_YEAR,
v.has_vpic_trailer_decode = true
RETURN count(t) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Append VPIC decode data to FARS Vehicle nodes.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument(
"--missing-mode",
choices=("empty", "null"),
default=os.getenv("FARS_MISSING_MODE", "empty"),
help=(
"Use 'empty' to store blank strings for missing columns. "
"Use 'null' to omit missing properties in Neo4j."
),
)
parser.add_argument("--skip-vpicdecode", action="store_true")
parser.add_argument("--skip-trailers", action="store_true")
parser.add_argument("--ensure-constraints-only", action="store_true")
parser.add_argument("--dry-run", action="store_true")
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def normalize_column(name: Any) -> str:
return (
str(name)
.replace("\ufeff", "")
.replace("", "")
.strip()
.upper()
)
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def prefixed_vpicdecode_column(column: str) -> str:
return f"VPICDECODE_{column}"
def find_files(base: Path, table: str) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
csv_path = year_dir / "National" / f"{table}.csv"
if csv_path.exists():
files.append((int(year_dir.name), csv_path))
return files
def read_header(csv_path: Path) -> list[str]:
df = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
return normalize_columns(df.columns)
def build_vpicdecode_columns(files: list[tuple[int, Path]]) -> list[str]:
columns: list[str] = []
seen: set[str] = set()
for _, csv_path in files:
for column in read_header(csv_path):
prefixed = prefixed_vpicdecode_column(column)
if prefixed not in seen:
seen.add(prefixed)
columns.append(prefixed)
for extra in ("vehicle_id", "accident_id", "vpicdecode_year", "vpicdecode_present"):
if extra not in seen:
columns.append(extra)
return columns
def build_trailer_columns(files: list[tuple[int, Path]]) -> list[str]:
columns: list[str] = []
seen: set[str] = set()
for _, csv_path in files:
for column in read_header(csv_path):
if column not in seen:
seen.add(column)
columns.append(column)
for extra in (
"trailer_id",
"vehicle_id",
"accident_id",
"DATA_YEAR",
"source_table",
):
if extra not in seen:
seen.add(extra)
columns.append(extra)
return columns
def count_rows(csv_path: Path) -> int:
with csv_path.open("r", encoding="latin-1", errors="replace", newline="") as fh:
return max(sum(1 for _ in fh) - 1, 0)
def trailer_id_stats(files: list[tuple[int, Path]]) -> tuple[int, int, int]:
total_rows = 0
trailer_ids: set[str] = set()
for year, csv_path in files:
df = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
usecols=["ST_CASE", "VEH_NO", "TRAILER_NO"],
)
df.columns = normalize_columns(df.columns)
st_case = df["ST_CASE"].map(identifier_part)
veh_no = df["VEH_NO"].map(identifier_part)
trailer_no = df["TRAILER_NO"].map(identifier_part)
ids = [
f"{year}_{case_number}_{vehicle_number}_{trailer_number}"
for case_number, vehicle_number, trailer_number in zip(st_case, veh_no, trailer_no)
]
total_rows += len(ids)
trailer_ids.update(ids)
return total_rows, len(trailer_ids), total_rows - len(trailer_ids)
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def clean_value(value: Any, missing_value: Any) -> Any:
if value is None:
return missing_value
if isinstance(value, list):
cleaned_list = []
for item in value:
cleaned_item = clean_value(item, missing_value)
if cleaned_item not in (None, ""):
cleaned_list.append(cleaned_item)
return cleaned_list
try:
if pd.isna(value):
return missing_value
except (TypeError, ValueError):
pass
if hasattr(value, "item") and not isinstance(value, str):
value = value.item()
if isinstance(value, str):
text = value.strip()
return text if text else missing_value
if isinstance(value, float):
if math.isnan(value):
return missing_value
if math.isfinite(value) and value.is_integer():
return int(value)
return float(value)
return value
def clean_row(row: dict[str, Any], missing_value: Any) -> dict[str, Any]:
return {key: clean_value(value, missing_value) for key, value in row.items()}
def complete_rows(
rows: list[dict[str, Any]],
master_columns: list[str],
missing_value: Any,
) -> list[dict[str, Any]]:
return [
clean_row(
{column: row.get(column, missing_value) for column in master_columns},
missing_value,
)
for row in rows
]
def add_vehicle_keys(chunk: pd.DataFrame, year: int) -> pd.DataFrame:
st_case = chunk["ST_CASE"].map(identifier_part)
veh_no = chunk["VEH_NO"].map(identifier_part)
chunk["accident_id"] = st_case.map(lambda value: f"{year}_{value}")
chunk["vehicle_id"] = [
f"{accident_id}_{vehicle_number}"
for accident_id, vehicle_number in zip(chunk["accident_id"], veh_no)
]
return chunk
def iter_vpicdecode_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(csv_path, encoding="latin-1", low_memory=False, chunksize=batch_size)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_vehicle_keys(chunk, year)
chunk["vpicdecode_year"] = year
chunk["vpicdecode_present"] = True
rename = {
column: prefixed_vpicdecode_column(column)
for column in chunk.columns
if column not in {"vehicle_id", "accident_id", "vpicdecode_year", "vpicdecode_present"}
}
chunk = chunk.rename(columns=rename)
chunk = chunk.reindex(columns=master_columns, fill_value=missing_value)
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [clean_row(row, missing_value) for row in chunk.to_dict(orient="records")]
def add_trailer_keys(chunk: pd.DataFrame, year: int) -> pd.DataFrame:
st_case = chunk["ST_CASE"].map(identifier_part)
veh_no = chunk["VEH_NO"].map(identifier_part)
trailer_no = chunk["TRAILER_NO"].map(identifier_part)
chunk["accident_id"] = st_case.map(lambda value: f"{year}_{value}")
chunk["vehicle_id"] = [
f"{accident_id}_{vehicle_number}"
for accident_id, vehicle_number in zip(chunk["accident_id"], veh_no)
]
chunk["trailer_id"] = [
f"{vehicle_id}_{trailer_number}"
for vehicle_id, trailer_number in zip(chunk["vehicle_id"], trailer_no)
]
return chunk
def iter_trailer_batches(
csv_path: Path,
year: int,
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(csv_path, encoding="latin-1", low_memory=False, chunksize=batch_size)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
chunk = add_trailer_keys(chunk, year)
chunk["DATA_YEAR"] = year
chunk["source_table"] = "VPICTRAILERDECODE"
chunk = chunk.reindex(columns=master_columns, fill_value=missing_value)
chunk = chunk.where(pd.notna(chunk), missing_value)
yield [clean_row(row, missing_value) for row in chunk.to_dict(orient="records")]
def quote_database_name(database: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.-]+", database):
raise ValueError(f"Unsafe Neo4j database name: {database!r}")
return f"`{database}`"
def get_databases(driver: Any) -> dict[str, str]:
with driver.session(database="system") as session:
result = session.run("SHOW DATABASES YIELD name, currentStatus RETURN name, currentStatus")
return {record["name"]: record["currentStatus"] for record in result}
def ensure_database(driver: Any, database: str) -> None:
status = get_databases(driver).get(database)
if status == "online":
return
raise SystemExit(f"Neo4j database {database!r} is not online; current status is {status!r}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE INDEX vehicle_vpicdecode_present IF NOT EXISTS FOR (v:Vehicle) ON (v.vpicdecode_present)",
":Vehicle(vpicdecode_present) index",
)
run_ddl(
session,
"CREATE CONSTRAINT trailer_id IF NOT EXISTS FOR (t:Trailer) REQUIRE t.trailer_id IS UNIQUE",
":Trailer(trailer_id) unique constraint",
)
run_ddl(
session,
"CREATE INDEX trailer_vehicle_id IF NOT EXISTS FOR (t:Trailer) ON (t.vehicle_id)",
":Trailer(vehicle_id) index",
)
run_ddl(
session,
"CREATE INDEX trailer_data_year IF NOT EXISTS FOR (t:Trailer) ON (t.DATA_YEAR)",
":Trailer(DATA_YEAR) index",
)
def execute_batch(session: Any, query: str, rows: list[dict[str, Any]]) -> int:
result = session.run(query, rows=rows)
record = result.single()
return int(record["matched"]) if record else 0
def print_dry_run(
vpic_files: list[tuple[int, Path]],
trailer_files: list[tuple[int, Path]],
vpic_columns: list[str],
trailer_columns: list[str],
) -> None:
print(f"VPICDECODE files : {len(vpic_files)}")
print(f"VPICDECODE rows : {sum(count_rows(path) for _, path in vpic_files):,}")
print(f"VPICDECODE properties : {len(vpic_columns)}")
print(f"VPICTRAILER files : {len(trailer_files)}")
trailer_rows, distinct_trailers, duplicate_trailers = trailer_id_stats(trailer_files)
print(f"VPICTRAILER rows : {trailer_rows:,}")
print(f"Trailer nodes : {distinct_trailers:,}")
print(f"Duplicate trailer IDs : {duplicate_trailers:,}")
print(f"Trailer properties : {len(trailer_columns)}")
table_years = Counter()
for year, _ in vpic_files:
table_years["vpicdecode"] += 1
for year, _ in trailer_files:
table_years["vpictrailerdecode"] += 1
print(f"File counts by table : {dict(table_years)}")
print(f"First VPIC properties : {', '.join(vpic_columns[:10])}")
print(f"Last VPIC properties : {', '.join(vpic_columns[-10:])}")
print(f"Trailer properties : {', '.join(trailer_columns[:10])} ... {', '.join(trailer_columns[-10:])}")
def append_vpicdecode(
session: Any,
files: list[tuple[int, Path]],
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> tuple[int, int]:
total_rows = 0
total_matched = 0
for index, (year, csv_path) in enumerate(files, 1):
start = time.time()
year_rows = 0
year_matched = 0
for rows in iter_vpicdecode_batches(csv_path, year, master_columns, batch_size, missing_value):
matched = execute_batch(session, UPDATE_VPICDECODE_QUERY, rows)
year_rows += len(rows)
year_matched += matched
total_rows += len(rows)
total_matched += matched
elapsed = time.time() - start
print(
f"[VPICDECODE {index}/{len(files)}] Done {year}: "
f"{year_matched:,}/{year_rows:,} matched in {seconds_text(elapsed)} ({format_duration(elapsed)})"
)
return total_rows, total_matched
def append_trailerdecode(
session: Any,
files: list[tuple[int, Path]],
master_columns: list[str],
batch_size: int,
missing_value: Any,
) -> tuple[int, int]:
total_rows = 0
total_matched = 0
for index, (year, csv_path) in enumerate(files, 1):
start = time.time()
year_rows = 0
year_matched = 0
for rows in iter_trailer_batches(csv_path, year, master_columns, batch_size, missing_value):
matched = execute_batch(session, IMPORT_TRAILER_QUERY, rows)
year_rows += len(rows)
year_matched += matched
total_rows += len(rows)
total_matched += matched
elapsed = time.time() - start
print(
f"[VPICTRAILER {index}/{len(files)}] Done {year}: "
f"{year_matched:,}/{year_rows:,} Trailer nodes matched in "
f"{seconds_text(elapsed)} ({format_duration(elapsed)})"
)
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
missing_value = "" if args.missing_mode == "empty" else None
vpic_files = [] if args.skip_vpicdecode else find_files(args.base, VPICDECODE_TABLE)
trailer_files = [] if args.skip_trailers else find_files(args.base, VPICTRAILERDECODE_TABLE)
vpic_columns = build_vpicdecode_columns(vpic_files) if vpic_files else []
trailer_columns = build_trailer_columns(trailer_files) if trailer_files else []
if args.dry_run:
print_dry_run(vpic_files, trailer_files, vpic_columns, trailer_columns)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
ensure_database(driver, args.database)
with driver.session(database=args.database) as session:
ensure_constraints(session)
if args.ensure_constraints_only:
print(f"Constraints/indexes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Missing values : {repr(missing_value)}")
print(f"Script started UTC : {utc_now()}")
print("")
vpic_rows = vpic_matched = 0
trailer_rows = trailer_matched = 0
if vpic_files:
vpic_start = time.time()
print(f"VPICDECODE started UTC : {utc_now()}")
vpic_rows, vpic_matched = append_vpicdecode(
session, vpic_files, vpic_columns, args.batch_size, missing_value
)
print(f"VPICDECODE seconds : {time.time() - vpic_start:.3f}")
print("")
if trailer_files:
trailer_start = time.time()
print(f"VPICTRAILER started UTC: {utc_now()}")
trailer_rows, trailer_matched = append_trailerdecode(
session, trailer_files, trailer_columns, args.batch_size, missing_value
)
print(f"VPICTRAILER seconds : {time.time() - trailer_start:.3f}")
total_elapsed = time.time() - total_start
print("")
print(f"Script finished UTC : {utc_now()}")
print(f"Total script seconds : {total_elapsed:.3f}")
print(f"VPICDECODE matched : {vpic_matched:,}/{vpic_rows:,}")
print(f"VPICTRAILER matched : {trailer_matched:,}/{trailer_rows:,} Trailer nodes")
finally:
driver.close()
if __name__ == "__main__":
main()
Creating Edges Between Core Nodes (220.524s)
ACCIDENT-[:INVOLVES]->VEHICLE: 41.224sACCIDENT-[:HAS_VICTIM]->PERSON: 63.819sACCIDENT-[:RESULTED_IN]->EVENT: 18.994sVEHICLE-[:HAS_OCCUPANT]->PERSON: 64.910sVEHICLE-[:SUBJECT_OF]->EVENT: 24.709sVEHICLE-[:OBJECT_OF]->EVENT: 6.868s
python
#!/usr/bin/env python3
"""
Create core FARS graph relationships in Neo4j.
This script assumes the node importers have already created:
:Accident(accident_id)
:Vehicle(vehicle_id, accident_id)
:Person(person_id, accident_id, vehicle_id)
:Event(event_id, accident_id)
Important: raw ST_CASE is not globally unique across years. All joins here use
the generated year-prefixed IDs so relationships cannot cross-link different
years that reused the same ST_CASE value.
Relationships created:
(:Accident)-[:INVOLVES]->(:Vehicle)
(:Accident)-[:HAS_VICTIM]->(:Person)
(:Accident)-[:RESULTED_IN]->(:Event)
(:Vehicle)-[:HAS_OCCUPANT]->(:Person)
(:Vehicle)-[:SUBJECT_OF]->(:Event)
(:Vehicle)-[:OBJECT_OF]->(:Event)
Vehicle-to-Event relationships here are based on CEVENT's VNUMBER1/VNUMBER2.
VEVENT should be imported separately later if we want vehicle-specific event
sequence order through VEVENTNUM.
"""
from __future__ import annotations
import argparse
import os
import re
import sys
import time
from datetime import datetime, timezone
from typing import Any
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
APOC_ITERATE_QUERY = """
CALL apoc.periodic.iterate(
$source,
$action,
{batchSize: $batch_size, parallel: false, iterateList: true}
)
YIELD batches, total, timeTaken, committedOperations, failedOperations, failedBatches, retries, errorMessages
RETURN batches, total, timeTaken, committedOperations, failedOperations, failedBatches, retries, errorMessages
"""
EDGES: list[dict[str, str]] = [
{
"label": "ACCIDENT-[:INVOLVES]->VEHICLE",
"source": """
MATCH (v:Vehicle)
WHERE v.accident_id IS NOT NULL AND v.accident_id <> ''
MATCH (a:Accident {accident_id: v.accident_id})
RETURN a, v
""",
"action": """
MERGE (a)-[r:INVOLVES]->(v)
SET r.source_key = 'vehicle.accident_id',
r.created_from = 'core_edges'
""",
"count_query": """
MATCH (v:Vehicle)
WHERE v.accident_id IS NOT NULL AND v.accident_id <> ''
MATCH (:Accident {accident_id: v.accident_id})
RETURN count(v) AS expected
""",
},
{
"label": "ACCIDENT-[:HAS_VICTIM]->PERSON",
"source": """
MATCH (p:Person)
WHERE p.accident_id IS NOT NULL AND p.accident_id <> ''
MATCH (a:Accident {accident_id: p.accident_id})
RETURN a, p
""",
"action": """
MERGE (a)-[r:HAS_VICTIM]->(p)
SET r.source_key = 'person.accident_id',
r.created_from = 'core_edges'
""",
"count_query": """
MATCH (p:Person)
WHERE p.accident_id IS NOT NULL AND p.accident_id <> ''
MATCH (:Accident {accident_id: p.accident_id})
RETURN count(p) AS expected
""",
},
{
"label": "ACCIDENT-[:RESULTED_IN]->EVENT",
"source": """
MATCH (e:Event)
WHERE e.accident_id IS NOT NULL AND e.accident_id <> ''
MATCH (a:Accident {accident_id: e.accident_id})
RETURN a, e
""",
"action": """
MERGE (a)-[r:RESULTED_IN]->(e)
SET r.source_key = 'event.accident_id',
r.created_from = 'core_edges'
""",
"count_query": """
MATCH (e:Event)
WHERE e.accident_id IS NOT NULL AND e.accident_id <> ''
MATCH (:Accident {accident_id: e.accident_id})
RETURN count(e) AS expected
""",
},
{
"label": "VEHICLE-[:HAS_OCCUPANT]->PERSON",
"source": """
MATCH (p:Person)
WHERE p.vehicle_id IS NOT NULL AND p.vehicle_id <> ''
MATCH (v:Vehicle {vehicle_id: p.vehicle_id})
RETURN v, p
""",
"action": """
MERGE (v)-[r:HAS_OCCUPANT]->(p)
SET r.source_key = 'person.vehicle_id',
r.created_from = 'core_edges'
""",
"count_query": """
MATCH (p:Person)
WHERE p.vehicle_id IS NOT NULL AND p.vehicle_id <> ''
MATCH (:Vehicle {vehicle_id: p.vehicle_id})
RETURN count(p) AS expected
""",
},
{
"label": "VEHICLE-[:SUBJECT_OF]->EVENT",
"source": """
MATCH (e:Event)
WITH e, toInteger(e.VNUMBER1) AS vehicle_number
WHERE vehicle_number >= 1 AND vehicle_number <= 999
MATCH (v:Vehicle {vehicle_id: e.accident_id + '_' + toString(vehicle_number)})
RETURN v, e, vehicle_number
""",
"action": """
MERGE (v)-[r:SUBJECT_OF]->(e)
SET r.role = 'this_vehicle',
r.vehicle_number = vehicle_number,
r.area_of_impact = e.AOI1,
r.eventnum = e.EVENTNUM,
r.DATA_YEAR = e.DATA_YEAR,
r.source_table = 'CEVENT',
r.source_columns = 'VNUMBER1,AOI1',
r.created_from = 'core_edges'
""",
"count_query": """
MATCH (e:Event)
WITH e, toInteger(e.VNUMBER1) AS vehicle_number
WHERE vehicle_number >= 1 AND vehicle_number <= 999
MATCH (:Vehicle {vehicle_id: e.accident_id + '_' + toString(vehicle_number)})
RETURN count(e) AS expected
""",
},
{
"label": "VEHICLE-[:OBJECT_OF]->EVENT",
"source": """
MATCH (e:Event)
WITH e, toInteger(e.VNUMBER2) AS vehicle_number
WHERE vehicle_number >= 1 AND vehicle_number <= 999
MATCH (v:Vehicle {vehicle_id: e.accident_id + '_' + toString(vehicle_number)})
RETURN v, e, vehicle_number
""",
"action": """
MERGE (v)-[r:OBJECT_OF]->(e)
SET r.role = 'other_vehicle',
r.vehicle_number = vehicle_number,
r.area_of_impact = e.AOI2,
r.eventnum = e.EVENTNUM,
r.DATA_YEAR = e.DATA_YEAR,
r.source_table = 'CEVENT',
r.source_columns = 'VNUMBER2,AOI2',
r.created_from = 'core_edges'
""",
"count_query": """
MATCH (e:Event)
WITH e, toInteger(e.VNUMBER2) AS vehicle_number
WHERE vehicle_number >= 1 AND vehicle_number <= 999
MATCH (:Vehicle {vehicle_id: e.accident_id + '_' + toString(vehicle_number)})
RETURN count(e) AS expected
""",
},
]
POST_COUNT_QUERIES: list[tuple[str, str]] = [
("INVOLVES", "MATCH ()-[r:INVOLVES]->() RETURN count(r) AS count"),
("HAS_VICTIM", "MATCH ()-[r:HAS_VICTIM]->() RETURN count(r) AS count"),
("RESULTED_IN", "MATCH ()-[r:RESULTED_IN]->() RETURN count(r) AS count"),
("HAS_OCCUPANT", "MATCH ()-[r:HAS_OCCUPANT]->() RETURN count(r) AS count"),
("SUBJECT_OF", "MATCH ()-[r:SUBJECT_OF]->() RETURN count(r) AS count"),
("OBJECT_OF", "MATCH ()-[r:OBJECT_OF]->() RETURN count(r) AS count"),
]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create core FARS relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Print expected relationship counts only.")
parser.add_argument(
"--ensure-indexes-only",
action="store_true",
help="Only ensure supporting indexes/constraints, then exit.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def quote_database_name(database: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_.-]+", database):
raise ValueError(f"Unsafe Neo4j database name: {database!r}")
return f"`{database}`"
def get_databases(driver: Any) -> dict[str, str]:
with driver.session(database="system") as session:
result = session.run("SHOW DATABASES YIELD name, currentStatus RETURN name, currentStatus")
return {record["name"]: record["currentStatus"] for record in result}
def ensure_database(driver: Any, database: str) -> None:
status = get_databases(driver).get(database)
if status == "online":
return
raise SystemExit(f"Neo4j database {database!r} is not online; current status is {status!r}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_indexes(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT accident_id IF NOT EXISTS FOR (a:Accident) REQUIRE a.accident_id IS UNIQUE",
":Accident(accident_id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT event_id IF NOT EXISTS FOR (e:Event) REQUIRE e.event_id IS UNIQUE",
":Event(event_id) unique constraint",
)
run_ddl(
session,
"CREATE INDEX vehicle_accident_id IF NOT EXISTS FOR (v:Vehicle) ON (v.accident_id)",
":Vehicle(accident_id) index",
)
run_ddl(
session,
"CREATE INDEX person_accident_id IF NOT EXISTS FOR (p:Person) ON (p.accident_id)",
":Person(accident_id) index",
)
run_ddl(
session,
"CREATE INDEX person_vehicle_id IF NOT EXISTS FOR (p:Person) ON (p.vehicle_id)",
":Person(vehicle_id) index",
)
run_ddl(
session,
"CREATE INDEX event_accident_id IF NOT EXISTS FOR (e:Event) ON (e.accident_id)",
":Event(accident_id) index",
)
def print_dry_run(session: Any) -> None:
print("Expected relationships from current node keys:")
for edge in EDGES:
start = time.time()
record = session.run(edge["count_query"]).single()
elapsed = time.time() - start
expected = int(record["expected"]) if record else 0
print(f" {edge['label']:<42} {expected:>12,} in {seconds_text(elapsed)}")
print("\nExisting relationship counts:")
for label, query in POST_COUNT_QUERIES:
record = session.run(query).single()
count = int(record["count"]) if record else 0
print(f" {label:<18} {count:>12,}")
def run_edge(session: Any, edge: dict[str, str], batch_size: int) -> dict[str, Any]:
result = session.run(
APOC_ITERATE_QUERY,
source=edge["source"],
action=edge["action"],
batch_size=batch_size,
)
record = result.single()
return dict(record) if record else {}
def import_edges(session: Any, batch_size: int) -> None:
for index, edge in enumerate(EDGES, 1):
start = time.time()
print(f"[{index}/{len(EDGES)}] Creating {edge['label']}...")
try:
stats = run_edge(session, edge, batch_size)
elapsed = time.time() - start
print(f" Done in {seconds_text(elapsed)} ({format_duration(elapsed)}) | {stats}")
except Exception as exc:
elapsed = time.time() - start
print(f" FAILED after {seconds_text(elapsed)} ({format_duration(elapsed)}): {exc}")
raise
def print_post_counts(session: Any) -> None:
print("\nRelationship counts:")
for label, query in POST_COUNT_QUERIES:
record = session.run(query).single()
count = int(record["count"]) if record else 0
print(f" {label:<18} {count:>12,}")
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
ensure_database(driver, args.database)
with driver.session(database=args.database) as session:
if args.dry_run:
print_dry_run(session)
return
ensure_indexes(session)
if args.ensure_indexes_only:
print(f"Indexes/constraints ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
edge_start = time.time()
print(f"Edge load started UTC: {utc_now()}")
import_edges(session, args.batch_size)
edge_elapsed = time.time() - edge_start
print(f"Edge load finished UTC: {utc_now()}")
print(f"Edge load seconds : {edge_elapsed:.3f}")
print_post_counts(session)
total_elapsed = time.time() - total_start
print(f"\nTotal script seconds: {total_elapsed:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Weather Nodes (50ms)
cypher
UNWIND [
{id: 1, name: 'Clear'},
{id: 2, name: 'Rain (Mist)'},
{id: 3, name: 'Sleet (Hail)'},
{id: 4, name: 'Snow'},
{id: 6, name: 'Severe Crosswinds'},
{id: 7, name: 'Blowing Sand, Soil, Dirt'},
{id: 8, name: 'Other'},
{id: 9, name: 'Cloudy'},
{id: 11, name: 'Blowing Snow'},
{id: 12, name: 'Freezing Rain or Drizzle'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Unknown'},
{id: 1001, name: 'Fog'},
{id: 1002, name: 'Smog, Smoke'}
] AS row
MERGE (:Weather {id: row.id, name: row.name});
CrashRF Nodes (218ms)
cypher
UNWIND [
{id: 1, name: 'Inadequate Warning of Exits, Lanes Narrowing, Traffic Controls, etc.'},
{id: 2, name: 'Shoulder Design or Condition'},
{id: 3, name: 'Other Maintenance or Construction-Created Condition'},
{id: 4, name: 'No or Obscured Pavement Marking'},
{id: 5, name: 'Surface Under Water'},
{id: 6, name: 'Inadequate Construction or Poor Design of Roadway, Bridge, etc.'},
{id: 7, name: 'Surface Washed Out (Caved in, Road Slippage)'},
{id: 10, name: 'Emergency Vehicle Related'},
{id: 12, name: 'Distracted Driver of a Non-Contact Vehicle'},
{id: 13, name: 'Aggressive Driving/Road Rage by Non-Contact Vehicle Driver'},
{id: 14, name: 'Motor Vehicle Struck by Falling Cargo or Something That Came Loose From or Something That Was Set in Motion by a Vehicle'},
{id: 15, name: 'Non-Occupant Struck by Falling Cargo, or Something Came Loose From or Something That Was Set in Motion by a Vehicle'},
{id: 16, name: 'Non-Occupant Struck Vehicle'},
{id: 17, name: 'Stopped Vehicle Set in Motion by Non-Driver'},
{id: 18, name: 'Date of Crash and Date of EMS Notification Were Not Same Day'},
{id: 19, name: 'Recent Previous Crash Scene Nearby'},
{id: 20, name: 'Police-Pursuit-Involved'},
{id: 21, name: 'Within Designated School Zone'},
{id: 22, name: 'Speed Limit Is a Statutory Limit as Recorded or Was Determined as This State Basic Rule'},
{id: 23, name: 'Indication of a Stalled/Disabled Vehicle'},
{id: 24, name: 'Unstabilized Situation Began and All Harmful Events Occurred off of the Roadway'},
{id: 25, name: 'Toll Booth/Plaza Related'},
{id: 26, name: 'Prior Non-Recurring Incident'},
{id: 27, name: 'Backup Due to Prior Crash'},
{id: 28, name: 'Regular Congestion'},
{id: 30, name: 'Obstructed Crosswalks'},
{id: 31, name: 'Related to a Bus Stop'},
{id: 40, name: 'Traffic Controls Not Functioning Properly'},
{id: 42, name: 'Uncontrolled Intersection or Railroad Crossing'},
{id: 99, name: 'Unknown'}
] AS row
MERGE (:CrashRF {id: row.id, name: row.name});
Damage Nodes (145ms)
cypher
UNWIND [
{id: 0, name: 'Non-Collision'},
{id: 1, name: 'Clock Point 1'},
{id: 2, name: 'Clock Point 2'},
{id: 3, name: 'Clock Point 3'},
{id: 4, name: 'Clock Point 4'},
{id: 5, name: 'Clock Point 5'},
{id: 6, name: 'Clock Point 6'},
{id: 7, name: 'Clock Point 7'},
{id: 8, name: 'Clock Point 8'},
{id: 9, name: 'Clock Point 9'},
{id: 10, name: 'Clock Point 10'},
{id: 11, name: 'Clock Point 11'},
{id: 12, name: 'Clock Point 12'},
{id: 13, name: 'Top'},
{id: 14, name: 'Undercarriage'},
{id: 18, name: 'Set-in-Motion (Not a Clock Point)'},
{id: 61, name: 'Left'},
{id: 62, name: 'Left-Front Half'},
{id: 63, name: 'Left-Back Half'},
{id: 81, name: 'Right'},
{id: 82, name: 'Right-Front Half'},
{id: 83, name: 'Right-Back Half'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Unknown'}
] AS row
MERGE (:Damage {id: row.id, name: row.name});
Distract Nodes (89ms)
cypher
UNWIND [
{id: 1, name: 'Looked but Did Not See'},
{id: 3, name: 'By Other Occupant(s)'},
{id: 4, name: 'By a Moving Object in Vehicle'},
{id: 5, name: 'While Talking or Listening to Mobile Phone'},
{id: 6, name: 'While Manipulating Mobile Phone'},
{id: 7, name: 'While Adjusting Audio or Climate Controls'},
{id: 9, name: 'While Using Other Component/Controls Integral to Vehicle'},
{id: 10, name: 'While Using or Reaching for Device/Object Brought Into Vehicle'},
{id: 12, name: 'Distracted by Outside Person, Object or Event'},
{id: 13, name: 'Eating or Drinking'},
{id: 14, name: 'Smoking Related'},
{id: 15, name: 'Other Mobile Phone Related'},
{id: 16, name: 'No Driver Present/Unknown if Driver Present'},
{id: 17, name: 'Distraction/Inattention'},
{id: 18, name: 'Distraction/Careless'},
{id: 19, name: 'Careless/Inattentive'},
{id: 92, name: 'Distraction (Distracted), Details Unknown'},
{id: 93, name: 'Inattention (Inattentive), Details Unknown'},
{id: 96, name: 'Not Reported'},
{id: 97, name: 'Lost in Thought/Daydreaming'},
{id: 98, name: 'Other Distraction'},
{id: 99, name: 'Reported as Unknown if Distracted'}
] AS row
MERGE (d:Distract {id: row.id})
SET d.name = row.name;
Factor Nodes (9304ms)
cypher
UNWIND [
{id: 1, name: 'Tires'},
{id: 2, name: 'Brake System'},
{id: 3, name: 'Steering'},
{id: 4, name: 'Suspension'},
{id: 5, name: 'Power Train'},
{id: 6, name: 'Exhaust System'},
{id: 7, name: 'Head Lights'},
{id: 8, name: 'Signal Lights'},
{id: 9, name: 'Other Lights'},
{id: 10, name: 'Wipers'},
{id: 11, name: 'Wheels'},
{id: 12, name: 'Mirrors'},
{id: 13, name: 'Windows/Windshield'},
{id: 14, name: 'Body, Doors'},
{id: 15, name: 'Truck Coupling/Trailer Hitch/Safety Chains'},
{id: 16, name: 'Safety Systems'},
{id: 17, name: 'Vehicle Contributing Factors - No Details'},
{id: 97, name: 'Other'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Unknown'},
{id: 1001, name: 'Horn'},
{id: 1002, name: 'Driver Seating and Control'},
{id: 1003, name: 'Air Bags'},
{id: 1004, name: 'Safety Belts'}
] AS row
MERGE (f:Factor {id: row.id})
SET f.name = row.name;
DrImpair Nodes (100ms)
cypher
UNWIND [
{id: 1, name: 'Ill, Blackout'},
{id: 2, name: 'Asleep or Fatigued'},
{id: 3, name: 'Walking With a Cane or Crutches, etc.'},
{id: 4, name: 'Paraplegic or Restricted to Wheelchair'},
{id: 5, name: 'Impaired Due to Previous Injury'},
{id: 6, name: 'Deaf/Hard of Hearing'},
{id: 7, name: 'Blind/Low Vision'},
{id: 8, name: 'Emotional (Depressed, Angry, Disturbed, etc.)'},
{id: 9, name: 'Under the Influence of Alcohol, Drugs, or Medication'},
{id: 10, name: 'Physical Impairment - No Details'},
{id: 95, name: 'No Driver Present/Unknown if Driver Present'},
{id: 96, name: 'Other Physical Impairment'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Unknown if Impaired'}
] AS row
MERGE (d:DrImpair {id: row.id})
SET d.name = row.name;
Maneuver Nodes (55ms)
cypher
UNWIND [
{id: 0, name: 'Driver Did Not Maneuver to Avoid'},
{id: 1, name: 'Object'},
{id: 2, name: 'Poor Road Conditions (Puddle, Ice, Pothole, etc.)'},
{id: 3, name: 'Live Animal'},
{id: 4, name: 'Contact Motor Vehicle (in This Crash)'},
{id: 5, name: 'Pedestrian, Pedalcyclist or Other Non-Motorist'},
{id: 92, name: 'Phantom/Non-Contact Motor Vehicle'},
{id: 95, name: 'No Driver Present/Unknown if Driver Present'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Reported as Unknown'}
] AS row
MERGE (m:Maneuver {id: row.id})
SET m.name = row.name;
Violatn Nodes (553ms)
cypher
CREATE CONSTRAINT violatn_id IF NOT EXISTS
FOR (v:Violatn) REQUIRE v.id IS UNIQUE;
UNWIND [
{id: 1, name: 'Manslaughter or Homicide'},
{id: 2, name: 'Willful Reckless Driving; Driving to Endanger; Negligent Driving'},
{id: 3, name: 'Unsafe Reckless (Not Willful, Wanton Reckless) Driving'},
{id: 4, name: 'Inattentive, Careless, Improper Driving, Driving Without Due Care'},
{id: 5, name: 'Fleeing or Eluding Police'},
{id: 6, name: 'Fail to Obey Police, Fireman, Authorized Person Directing Traffic'},
{id: 7, name: 'Hit-And-Run, Fail to Stop After Crash'},
{id: 8, name: 'Fail to Give Aid, Information, Wait for Police After Crash'},
{id: 9, name: 'Serious Violation Resulting in Death'},
{id: 10, name: 'Use of Telecommunications Device'},
{id: 11, name: 'Driving While Intoxicated (Alcohol or Drugs) or BAC Above Limit'},
{id: 12, name: 'Driving While Impaired'},
{id: 13, name: 'Driving Under Influence of Substance Not Intended to Intoxicate'},
{id: 14, name: 'Drinking While Operating'},
{id: 15, name: 'Illegal Possession of Alcohol or Drugs'},
{id: 16, name: 'Driving With Detectable Alcohol'},
{id: 18, name: 'Refusal to Submit to Chemical Test'},
{id: 19, name: 'Alcohol, Drug or Impairment Violations Generally'},
{id: 21, name: 'Racing'},
{id: 22, name: 'Speeding (Above the Speed Limit)'},
{id: 23, name: 'Speed Greater Than Reasonable and Prudent'},
{id: 24, name: 'Exceeding Special Speed Limit'},
{id: 25, name: 'Energy Speed (Exceeding 55 mph, Non-Pointable)'},
{id: 26, name: 'Driving Too Slowly'},
{id: 29, name: 'Speed-Related Violations Generally'},
{id: 31, name: 'Fail to Stop for Red Signal'},
{id: 32, name: 'Fail to Stop for Flashing Red'},
{id: 33, name: 'Violation of Turn on Red'},
{id: 34, name: 'Fail to Obey Flashing Signal (Yellow or Red)'},
{id: 35, name: 'Fail to Obey Signal Generally'},
{id: 36, name: 'Violate RR Grade Crossing Device/Regulations'},
{id: 37, name: 'Fail to Obey Stop Sign'},
{id: 38, name: 'Fail to Obey Yield Sign'},
{id: 39, name: 'Fail to Obey Traffic Control Device Generally'},
{id: 41, name: 'Turn in Violation of Traffic Control'},
{id: 42, name: 'Improper Method and Position of Turn (Too Wide, Wrong Lane)'},
{id: 43, name: 'Fail to Signal for Turn or Stop'},
{id: 45, name: 'Fail to Yield to Emergency Vehicle'},
{id: 46, name: 'Fail to Yield Generally'},
{id: 48, name: 'Enter Intersection When Space Insufficient'},
{id: 49, name: 'Turn, Yield, Signaling Violations Generally'},
{id: 51, name: 'Driving Wrong Way on One-Way Road'},
{id: 52, name: 'Driving on Left, Wrong Side of Road Generally'},
{id: 53, name: 'Improper, Unsafe Passing'},
{id: 54, name: 'Pass on Right (Drive off Pavement to Pass)'},
{id: 55, name: 'Pass Stopped School Bus'},
{id: 56, name: 'Fail to Give Way When Overtaken'},
{id: 58, name: 'Following Too Closely'},
{id: 59, name: 'Wrong Side, Passing, Following Violations Generally'},
{id: 61, name: 'Unsafe or Prohibited Lane Change'},
{id: 62, name: 'Improper Use of Lane'},
{id: 63, name: 'Certain Traffic to Use Right Lane'},
{id: 66, name: 'Motorcycle Lane Violations'},
{id: 67, name: 'Motorcyclist Attached to Another Vehicle'},
{id: 69, name: 'Lane Violations Generally'},
{id: 71, name: 'Driving While License Withdrawn'},
{id: 72, name: 'Other Driver License Violations'},
{id: 73, name: 'Commercial Driver Violations'},
{id: 74, name: 'Vehicle Registration Violations'},
{id: 75, name: 'Fail to Carry Insurance Card'},
{id: 76, name: 'Driving Uninsured Vehicle'},
{id: 79, name: 'Non-Moving Violations Generally'},
{id: 81, name: 'Lamp Violations'},
{id: 82, name: 'Brake Violations'},
{id: 83, name: 'Failure to Require Restraint Use'},
{id: 84, name: 'Motorcycle Equipment Violations'},
{id: 85, name: 'Violation of Hazardous Cargo Regulations'},
{id: 86, name: 'Size, Weight, Load Violations'},
{id: 89, name: 'Equipment Violations Generally'},
{id: 91, name: 'Parking'},
{id: 92, name: 'Theft, Unauthorized Use of Motor Vehicle'},
{id: 93, name: 'Driving Where Prohibited'},
{id: 95, name: 'No Driver Present/Unknown if Driver Present'},
{id: 97, name: 'Not Reported'},
{id: 98, name: 'Other Moving Violation'},
{id: 99, name: 'Unknown Violations'},
{id: 1001, name: 'Yes (1975-1981)'}
] AS row
MERGE (v:Violatn {id: row.id})
SET v.name = row.name;
Vision Nodes (204ms)
cypher
UNWIND [
{id: 1, name: 'Rain, Snow, Fog, Smoke, Sand, Dust'},
{id: 2, name: 'Reflected Glare, Bright Sunlight, Headlights'},
{id: 3, name: 'Curve, Hill, or Other Roadway Design Features'},
{id: 4, name: 'Building, Billboard, or Other Structure'},
{id: 5, name: 'Trees, Crops, Vegetation'},
{id: 6, name: 'In-Transport Motor Vehicle (Including Load)'},
{id: 7, name: 'Not In-Transport Motor Vehicle (Parked, Working)'},
{id: 8, name: 'Splash or Spray of Passing Vehicle'},
{id: 9, name: 'Inadequate Defrost or Defog System'},
{id: 10, name: 'Inadequate Vehicle Lighting System'},
{id: 11, name: 'Obstructing Interior to the Vehicle'},
{id: 12, name: 'External Mirrors'},
{id: 13, name: 'Broken or Improperly Cleaned Windshield'},
{id: 14, name: 'Obstructing Angles on Vehicle'},
{id: 95, name: 'No Driver Present/Unknown if Driver Present'},
{id: 97, name: 'Vision Obscured - No Details'},
{id: 98, name: 'Other Visual Obstruction'},
{id: 99, name: 'Reported as Unknown'},
{id: 1001, name: 'Head Restraints'}
] AS row
MERGE (v:Vision {id: row.id})
SET v.name = row.name;
VehicleSF Nodes (271ms)
cypher
UNWIND [
{id: 1, name: 'Tires (Does Not Include Wheels, See Value 16)'},
{id: 2, name: 'Brake System'},
{id: 3, name: 'Steering System-Tie Rod, Kingpin, Ball Joint, etc.'},
{id: 4, name: 'Suspension-Springs, Shock Absorbers, MacPherson Struts, Axle Bearing, Control Arms, etc.'},
{id: 5, name: 'Power Train (Power Train/Engine, 2001-2009)-Universal Joint, Drive Shaft, Transmission, etc.'},
{id: 6, name: 'Exhaust System'},
{id: 7, name: 'Headlights'},
{id: 8, name: 'Signal Lights'},
{id: 9, name: 'Other Lights'},
{id: 10, name: 'Horn'},
{id: 11, name: 'Mirrors'},
{id: 12, name: 'Wipers'},
{id: 13, name: 'Driver Seating and Control'},
{id: 14, name: 'Body, Doors, Hood, Other'},
{id: 15, name: 'Trailer Hitch'},
{id: 16, name: 'Wheels'},
{id: 17, name: 'Air Bags'},
{id: 18, name: 'Other Vehicle Defects'},
{id: 19, name: 'Safety Belts'},
{id: 29, name: 'Default Code Used for Vehicle Numbering'},
{id: 30, name: 'Multi-Wheeled Motorcycle Conversion'},
{id: 31, name: 'Hit-and-Run Vehicle'},
{id: 32, name: 'Vehicle Registration for a Person with a Disability'},
{id: 33, name: 'Vehicle Being Pushed by Non-Motorist'},
{id: 34, name: 'Vehicle Impact Point-the Result of Something Set in Motion'},
{id: 35, name: 'Reconstructed/Altered Vehicle'},
{id: 37, name: 'Transporting Children to/From Head Start/Day Care'},
{id: 38, name: 'Vehicle Went Airborne During Crash'},
{id: 39, name: 'Highway Construction, Maintenance, or Utility Vehicle, In-Transport (Inside or Outside Work Zone)'},
{id: 40, name: 'Highway Incident Response Vehicle'},
{id: 41, name: 'Police, Fire, or EMS Vehicle Working at the Scene of an Emergency or Performing Other Traffic Control Activities'},
{id: 42, name: 'Other Working Vehicle (Not Construction, Maintenance, Utility, Police, Fire, or EMS Vehicle)'},
{id: 43, name: 'Hazardous Materials/Cargo Released From This Vehicle'},
{id: 44, name: 'Adaptive Equipment'},
{id: 45, name: 'Slide-in Camper'},
{id: 99, name: 'Unknown'},
{id: 999, name: 'Unknown'}
] AS row
MERGE (sf:VehicleSF {id: row.id})
SET sf.name = row.name;
DriverRF Nodes (289ms)
cypher
UNWIND [
{id: 4, name: 'Reaction to or Failure to Take Drugs/Medication'},
{id: 5, name: 'Other Drugs (Marijuana, Cocaine, etc.)'},
{id: 6, name: 'Careless Driving, Inattentive Operation, Improper Driving, Driving Without Due Care'},
{id: 8, name: 'Road Rage/Aggressive Driving'},
{id: 9, name: 'Emergency Services Personnel'},
{id: 10, name: 'Looked but Did Not See'},
{id: 12, name: 'Mother of Dead Fetus/Mother of Infant Born Post Crash'},
{id: 13, name: 'Mentally Challenged'},
{id: 14, name: 'Failure to Take Drugs/Medication'},
{id: 15, name: 'Seat Back Not in Normal Position, Seat Back Reclined'},
{id: 16, name: 'Police or Law Enforcement Officer'},
{id: 17, name: 'Running off Road'},
{id: 18, name: 'Traveling on Prohibited Trafficways'},
{id: 19, name: 'Legally Driving on Suspended or Revoked License'},
{id: 20, name: 'Leaving Vehicle Unattended With Engine Running; Leaving Vehicle Unattended in Roadway'},
{id: 21, name: 'Overloading or Improper Loading of Vehicle With Passenger or Cargo'},
{id: 22, name: 'Towing or Pushing Vehicle Improperly'},
{id: 23, name: 'Failing to Dim Lights or to Have Lights on When Required'},
{id: 24, name: 'Operating Without Required Equipment'},
{id: 25, name: 'Creating Unlawful Noise or Using Equipment Prohibited by Law'},
{id: 26, name: 'Following Improperly'},
{id: 27, name: 'Improper or Erratic Lane Changing'},
{id: 28, name: 'Improper Lane Usage'},
{id: 29, name: 'Intentional Illegal Driving off the Roadway'},
{id: 30, name: 'Making Improper Entry to or Exit From Trafficway'},
{id: 31, name: 'Starting or Backing Improperly'},
{id: 32, name: 'Opening Vehicle Closure Into Moving Traffic or Vehicle Is in Motion'},
{id: 33, name: 'Passing Where Prohibited by Posted Signs, Pavement Markings, or School Bus Displaying Warning Not to Pass'},
{id: 34, name: 'Improper Passing Location'},
{id: 35, name: 'Passing With Insufficient Distance or Inadequate Visibility or Failing to Yield to Overtaking Vehicle'},
{id: 36, name: 'Operating the Vehicle in an Erratic, Reckless, Careless or Negligent Manner'},
{id: 37, name: 'Police Pursuing This Driver or Police Officer in Pursuit'},
{id: 38, name: 'Failure to Yield Right-of-Way'},
{id: 39, name: 'Failure to Obey Actual Traffic Signs, Traffic Control Devices or Traffic Officers, Failure to Observe Safety Zone Traffic Laws'},
{id: 40, name: 'Passing Through or Around Barrier'},
{id: 41, name: 'Failure to Observe Warnings or Instructions on Vehicle Displaying Them'},
{id: 42, name: 'Failure to Signal Intentions'},
{id: 43, name: 'Giving Wrong Signal'},
{id: 44, name: 'Driving Too Fast for Conditions or in Excess of Posted Speed Limit'},
{id: 45, name: 'Driving Less Than Posted Maximum'},
{id: 46, name: 'Operating at Erratic or Suddenly Changing Speeds'},
{id: 47, name: 'Making Right Turn From Left-Turn Lane or Making Left Turn From Right-Turn Lane'},
{id: 48, name: 'Making Improper Turn'},
{id: 49, name: 'Failure to Comply With Physical Restrictions of License'},
{id: 50, name: 'Driving Wrong Way on One-Way Trafficway'},
{id: 51, name: 'Driving on Wrong Side of Two-way Trafficway (Intentionally or Unintentionally)'},
{id: 52, name: 'Operator Inexperience'},
{id: 53, name: 'Unfamiliar With Roadway'},
{id: 54, name: 'Stopping in Roadway (Vehicle Not Abandoned)'},
{id: 55, name: 'Improper Management of Vehicle Controls'},
{id: 56, name: 'Object Interference With Vehicle Controls'},
{id: 57, name: 'Driving With Tire-Related Problems'},
{id: 58, name: 'Over Correcting'},
{id: 59, name: 'Getting off/out of a Vehicle'},
{id: 60, name: 'Alcohol and/or Drug Test Refused'},
{id: 73, name: 'Driver Has Not Complied With Learners Permit or Intermediate Driver License Restrictions (GDL Restrictions)'},
{id: 74, name: 'Driver Has Not Complied With Physical or Other Imposed Restrictions'},
{id: 77, name: 'Severe Crosswind'},
{id: 78, name: 'Wind From Passing Truck'},
{id: 79, name: 'Slippery or Loose Surface'},
{id: 80, name: 'Tire Blow-Out or Flat'},
{id: 81, name: 'Debris or Objects in Road'},
{id: 82, name: 'Ruts, Holes, Bumps in Road'},
{id: 83, name: 'Live Animals in Road'},
{id: 84, name: 'Vehicle in Road'},
{id: 85, name: 'Phantom Vehicle'},
{id: 86, name: 'Pedestrian, Pedalcyclist, or Other Non-Motorist in Road'},
{id: 87, name: 'Ice, Water, Snow, Slush, Sand, Dirt, Oil, Wet Leaves on Road'},
{id: 88, name: 'Trailer Fishtailing or Swaying'},
{id: 89, name: "Driver has a Driving Record or Driver's License From More Than One State"},
{id: 90, name: 'Hit-and-Run Vehicle Driver'},
{id: 91, name: 'Non-Traffic Violation Charged (Manslaughter, Homicide or Other Assault Offense Committed Without Malice)'},
{id: 92, name: 'Other Non-Moving Traffic Violation'},
{id: 93, name: 'Cellular Telephone'},
{id: 94, name: 'Emergency Medical Service Personnel'},
{id: 95, name: 'Fire Personnel'},
{id: 96, name: 'Tow Operator'},
{id: 97, name: 'Transportation (i.e., Maintenance Workers, Safety Service Patrol Operators, etc.)'},
{id: 98, name: 'Head-Up Display (1991-2009)'},
{id: 999, name: 'Reported as Unknown'}
] AS row
MERGE (rf:DriverRF {id: row.id})
SET rf.name = row.name;
Race Nodes (185ms)
cypher
UNWIND [
{id: 0, name: 'Not a Fatality (Not Applicable)'},
{id: 1, name: 'White'},
{id: 2, name: 'Black or African American'},
{id: 3, name: 'North American Indian or Alaska Native'},
{id: 4, name: 'Chinese'},
{id: 5, name: 'Japanese'},
{id: 6, name: 'Native Hawaiian'},
{id: 7, name: 'Filipino'},
{id: 18, name: 'Asian Indian'},
{id: 19, name: 'Other Indian'},
{id: 28, name: 'Korean'},
{id: 38, name: 'Samoan'},
{id: 48, name: 'Vietnamese'},
{id: 58, name: 'Guamanian or Chamorro'},
{id: 68, name: 'Other Asian or Pacific Islander'},
{id: 78, name: 'Asian or Pacific Islander, No Specific (Individual) Race'},
{id: 96, name: 'Redacted'},
{id: 97, name: 'Multiple Races, Unspecified'},
{id: 98, name: 'All Other Races'},
{id: 99, name: 'Unknown'}
] AS row
MERGE (:Race {id: row.id, name: row.name});
PersonRF Nodes (326ms)
cypher
UNWIND [
{id: 4, name: 'Walking/Riding With or Against Traffic, Playing, Working, Sitting, Lying, Standing, etc., in Roadway'},
{id: 5, name: 'Interfering With Driver*'},
{id: 8, name: 'Person with an Intellectual, Cognitive, or Developmental Disability'},
{id: 9, name: 'Construction/Maintenance/Utility Worker'},
{id: 10, name: 'Alcohol and/or Drug Test Refused (Since 2017)'},
{id: 13, name: 'Motorized Wheelchair Rider**'},
{id: 18, name: 'Mother of Dead Fetus/Mother of Infant Born Post Crash'},
{id: 19, name: 'Pedestrian'},
{id: 20, name: 'Leaving Vehicle Unattended in Roadway'},
{id: 21, name: 'Overloading or Improper Loading of Vehicle With Passengers or Cargo'},
{id: 22, name: 'Towing or Pushing Vehicle Improperly'},
{id: 24, name: 'Operating Without Required Equipment'},
{id: 25, name: 'Creating Unlawful Noise or Using Equipment Prohibited by Law'},
{id: 26, name: 'Following Improperly'},
{id: 29, name: 'Illegal Driving on Road Shoulder, in Ditch, on Sidewalk, on Median'},
{id: 31, name: 'Default Code Used for Vehicle Numbering**'},
{id: 32, name: 'Opening Vehicle Closure Into Moving Traffic or While Vehicle is in Motion (Since 2001)*'},
{id: 33, name: 'Passing Where Prohibited by Posted Signs, Pavement Markings, Hill, or Curve, or School Bus Displaying Warning Not to Pass'},
{id: 34, name: 'Passing on Wrong Side'},
{id: 36, name: 'Operating the Vehicle in Other Erratic, Reckless, Careless or Negligent Manner'},
{id: 37, name: 'Traveling on Prohibited Trafficway'},
{id: 39, name: 'Failure to Obey Actual Traffic Signs, Traffic Control Devices or Traffic Officers; Failure to Obey Safety Zone Traffic Laws'},
{id: 40, name: 'Passing Through or Around Barrier Positioned to Prohibit or Channel Traffic'},
{id: 41, name: 'Failure to Observe Warnings or Instructions on Vehicles Displaying Them'},
{id: 42, name: 'Failure to Signal Intentions'},
{id: 43, name: 'Giving Wrong Signal'},
{id: 44, name: 'Driving Too Fast for Conditions or in Excess of Posted Speed Limit'},
{id: 45, name: 'Driving Less Than Posted Maximum'},
{id: 46, name: 'Operating at Erratic or Suddenly Changing Speeds'},
{id: 47, name: 'Making Right Turn From Left Turn Lane or Making Left Turn From Right Turn Lane'},
{id: 51, name: 'Operator Inexperience'},
{id: 52, name: 'Unfamiliar With Roadway'},
{id: 53, name: 'Non-Motorist Previously Used a Motor Vehicle for Motion**'},
{id: 54, name: 'Non-Motorist Attempting to Use a Motor Vehicle for Motion**'},
{id: 55, name: 'Non-Motorist Attempting to Use or Previously Used a Motor Vehicle for Motion, Details Not Reported**'},
{id: 56, name: 'Non-Driver Flees Scene (Since 2005)'},
{id: 57, name: 'Improper Tire Pressure (Since 1995)'},
{id: 58, name: 'Locked Wheel (1995-2014)'},
{id: 59, name: 'Overcorrecting (Since 1995)*'},
{id: 60, name: 'Rain, Snow, Fog, Smoke, Sand, Dust'},
{id: 61, name: 'Reflected Glare, Bright Sunlight, Headlights'},
{id: 62, name: 'Curve, Hill, or Other Design Features (Including Traffic Signs, Embankment)'},
{id: 63, name: 'Building, Billboard, Other Structures'},
{id: 64, name: 'Trees, Crops, Vegetation'},
{id: 65, name: 'Motor Vehicle (Including Load)'},
{id: 66, name: 'Parked Vehicle'},
{id: 67, name: 'Splash or Spray or Passing Vehicle'},
{id: 68, name: 'Inadequate Lighting System'},
{id: 69, name: 'Obstructing Angles on Vehicle'},
{id: 70, name: 'Mirrors'},
{id: 71, name: 'Mirrors-Other'},
{id: 72, name: 'Other Visual Obstruction'},
{id: 73, name: 'Severe Crosswind'},
{id: 74, name: 'Wind From Passing Truck'},
{id: 75, name: 'Slippery or Loose Surface'},
{id: 76, name: 'Tire Blow-Out or Flat'},
{id: 77, name: 'Debris or Objects in Road'},
{id: 78, name: 'Ruts, Holes, Bumps in Road'},
{id: 79, name: 'Live Animals in Road'},
{id: 80, name: 'Vehicle in Road'},
{id: 81, name: 'Phantom Vehicle'},
{id: 82, name: 'Pedestrian, Pedalcyclist, or Other Non-Motorist'},
{id: 83, name: 'Ice, Snow, Slush, Water, Sand, Dirt, Oil, Wet Leaves on Road (Since 1995)'},
{id: 86, name: 'Emergency Services Personnel (Since 2007)'},
{id: 87, name: 'Police or Law Enforcement Officer'},
{id: 88, name: 'Seat Back Not in Normal Upright Position, Seat Back Reclined*'},
{id: 89, name: 'Parked Motor Vehicle With Equipment Extending Into the Travel Lane'},
{id: 90, name: 'Non-Motorist Pushing a Vehicle**'},
{id: 91, name: 'Portable Electronic Devices'},
{id: 92, name: 'Person in Ambulance Treatment Compartment*'},
{id: 93, name: 'Non-Motorist Wearing Motorcycle Helmet**'},
{id: 94, name: 'Emergency Medical Services Personnel'},
{id: 95, name: 'Fire Personnel'},
{id: 96, name: 'Tow Operator'},
{id: 97, name: 'Transportation (Maintenance Workers, Safety Service Patrol Operators, etc.)'},
{id: 100, name: 'Using a Shared Micromobility Device**'},
{id: 101, name: 'Obstructed Sidewalk (for this Person)**'},
{id: 999, name: 'Reported as Unknown'},
{id: 1001, name: 'Stopping in Roadway (Vehicle Not Abandoned)'},
{id: 1002, name: 'Underriding a Parked Truck (1982-1996)'},
{id: 1003, name: 'Getting off/out of or on/into Moving Transport Vehicle'}
] AS row
MERGE (n:PersonRF {id: row.id})
SET n.name = row.name;
NMDistract Nodes (72ms)
cypher
UNWIND [
{id: 2, name: 'By Other Non-Motorist(s)'},
{id: 3, name: 'By a Driver or Occupant of a Motor Vehicle'},
{id: 5, name: 'While Talking or Listening to Mobile Phone'},
{id: 6, name: 'While Manipulating Mobile Phone'},
{id: 7, name: 'Adjusting or Listening to Portable Audio Device (Other Than on a Mobile Phone)'},
{id: 8, name: 'Adjusting, Talking to, or Manipulating Other Portable Electronic Device'},
{id: 12, name: 'Distracted by Animal, Other Object, Event, or Activity'},
{id: 13, name: 'Eating or Drinking'},
{id: 14, name: 'Smoking Related'},
{id: 15, name: 'Other Mobile Phone Related'},
{id: 17, name: 'Distraction/Inattention'},
{id: 18, name: 'Distraction/Careless'},
{id: 19, name: 'Careless/Inattentive'},
{id: 92, name: 'Distraction (Distracted), Details Unknown'},
{id: 93, name: 'Inattention (Inattentive), Details Unknown'},
{id: 96, name: 'Not Reported'},
{id: 97, name: 'Lost in Thought/Daydreaming'},
{id: 98, name: 'Other Distraction'},
{id: 99, name: 'Reported as Unknown if Distracted'}
] AS row
MERGE (n:NMDistract {id: row.id})
SET n.name = row.name;
NMCrash Nodes (524ms)
cypher
UNWIND [
{id: 1, name: 'Dart-out - Visual Obstruction Noted'},
{id: 2, name: 'Failure to Yield Right-Of-Way'},
{id: 3, name: 'Failure to Obey Traffic Signs, Signals or Officer'},
{id: 4, name: 'In Roadway Improperly (Standing, Lying, Working, Playing)'},
{id: 5, name: 'Entering/Exiting Parked or Stopped Vehicle'},
{id: 6, name: 'Inattentive (Talking, Eating, etc.)'},
{id: 7, name: 'Improper Turn/Merge'},
{id: 8, name: 'Improper Passing'},
{id: 9, name: 'Wrong-Way Riding or Walking'},
{id: 10, name: 'Riding on Wrong Side of Road'},
{id: 11, name: 'Dash - Run, No Visual Obstruction Noted'},
{id: 12, name: 'Improper Crossing of Roadway or Intersection (Jaywalking)'},
{id: 13, name: 'Failing to Have Lights on When Required'},
{id: 14, name: 'Operating Without Required Equipment'},
{id: 15, name: 'Improper or Erratic Lane Changing'},
{id: 16, name: 'Failure to Keep in Proper Lane or Running off Road'},
{id: 17, name: 'Making Improper Entry to or Exit From Trafficway'},
{id: 18, name: 'Operating in Other Erratic, Reckless, Careless or Negligent Manner'},
{id: 19, name: 'Not Visible (Dark Clothing, No Lighting, etc.)'},
{id: 20, name: 'Passing With Insufficient Distance or Inadequate Visibility or Failing to Yield to Overtaking Vehicle'},
{id: 21, name: 'Other'},
{id: 92, name: 'Contributing Circumstance -No Details'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Reported as Unknown'}
] AS row
MERGE (n:NMCrash {id: row.id})
SET n.name = row.name;
NMPrior Nodes (112ms)
cypher
UNWIND [
{id: 1, name: 'Going to or From School [Pre-K-12] (Since 2023)'},
{id: 2, name: 'Waiting to Cross Roadway'},
{id: 3, name: 'Crossing Roadway'},
{id: 4, name: 'Jogging/Running'},
{id: 5, name: 'Movement Along Roadway With Traffic (in or Adjacent to Travel Lane)'},
{id: 6, name: 'Movement Along Roadway Against Traffic (in or Adjacent to Travel Lane)'},
{id: 7, name: 'Movement on Sidewalk'},
{id: 8, name: 'In Roadway-Other [Working, Playing, etc.]'},
{id: 9, name: 'Adjacent to Roadway (e.g., Shoulder, Median)'},
{id: 10, name: 'Working in Trafficway [Incident Response]'},
{id: 11, name: 'Entering/Exiting a Vehicle'},
{id: 12, name: 'Disabled Vehicle Related (Working on, Pushing, Leaving/Approaching)'},
{id: 14, name: 'Other'},
{id: 15, name: 'None'},
{id: 16, name: 'Movement Along Roadway - Direction Unknown (Since 2012)'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Unknown'}
] AS row
MERGE (n:NMPrior {id: row.id})
SET n.name = row.name;
NMImpair Nodes (107ms)
cypher
UNWIND [
{id: 1, name: 'Ill, Blackout'},
{id: 2, name: 'Asleep or Fatigued'},
{id: 3, name: 'Walking With a Cane or Crutches'},
{id: 4, name: 'Paraplegic or in a Wheelchair'},
{id: 5, name: 'Impaired Due to Previous Injury'},
{id: 6, name: 'Deaf/Hard of Hearing'},
{id: 7, name: 'Blind/Low Vision'},
{id: 8, name: 'Emotional (Depressed, Angry, Disturbed, etc.)'},
{id: 9, name: 'Under the Influence of Alcohol, Drugs, or Medication'},
{id: 10, name: 'Physical Impairment - No Details'},
{id: 96, name: 'Other Physical Impairment'},
{id: 98, name: 'Not Reported'},
{id: 99, name: 'Reported as Unknown if Impaired'}
] AS row
MERGE (n:NMImpair {id: row.id})
SET n.name = row.name;
DrugTest Nodes (178ms)
cypher
UNWIND [
{id: 1, name: 'Whole Blood'},
{id: 2, name: 'Urine'},
{id: 11, name: 'Blood Plasma/Serum'},
{id: 12, name: 'Blood Clot'},
{id: 13, name: 'Oral Fluids'},
{id: 14, name: 'Vitreous'},
{id: 15, name: 'Liver'},
{id: 96, name: 'Not Reported'},
{id: 97, name: 'Unknown Specimen'},
{id: 98, name: 'Other Specimen'},
{id: 99, name: 'Reported as Unknown if Tested'}
] AS row
MERGE (n:DrugTest {id: row.id})
SET n.name = row.name;
CrashRF to Accident Relationship (20.233s)
Average edge import time per year: 0.446s across 1975-1981.
Average edge import time per year: 0.359s across 1982-2012.
Average edge import time per year: 0.687s across 2013-2017.
Average edge import time per year: 0.511s across 2018-2022.
python
#!/usr/bin/env python3
"""
Create FARS CrashRF-to-Accident relationships in Neo4j.
Sources:
* 1975-2019: Accident.CF1, Accident.CF2, Accident.CF3
* 2020-2022: crashrf.csv, one row per crash-related factor
The graph join uses accident_id = <year>_<ST_CASE>. Raw ST_CASE is not globally
unique across years, so it must not be used for Neo4j joins.
Relationship:
(:CrashRF)-[:CRASH_FACTOR_OF]->(:Accident)
For legacy 1975-1981 CF codes, only values that can be mapped into the modern
CrashRF category nodes are imported. Weather-related legacy code 20 is excluded
because it is handled by the Weather importer.
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
CRASHRF_NODES: dict[int, str] = {
1: "Inadequate Warning of Exits, Lanes Narrowing, Traffic Controls, etc.",
2: "Shoulder Design or Condition",
3: "Other Maintenance or Construction-Created Condition",
4: "No or Obscured Pavement Marking",
5: "Surface Under Water",
6: "Inadequate Construction or Poor Design of Roadway, Bridge, etc.",
7: "Surface Washed Out (Caved in, Road Slippage)",
10: "Emergency Vehicle Related",
12: "Distracted Driver of a Non-Contact Vehicle",
13: "Aggressive Driving/Road Rage by Non-Contact Vehicle Driver",
14: "Motor Vehicle Struck by Falling Cargo or Something That Came Loose From or Something That Was Set in Motion by a Vehicle",
15: "Non-Occupant Struck by Falling Cargo, or Something Came Loose From or Something That Was Set in Motion by a Vehicle",
16: "Non-Occupant Struck Vehicle",
17: "Stopped Vehicle Set in Motion by Non-Driver",
18: "Date of Crash and Date of EMS Notification Were Not Same Day",
19: "Recent Previous Crash Scene Nearby",
20: "Police-Pursuit-Involved",
21: "Within Designated School Zone",
22: "Speed Limit Is a Statutory Limit as Recorded or Was Determined as This State Basic Rule",
23: "Indication of a Stalled/Disabled Vehicle",
24: "Unstabilized Situation Began and All Harmful Events Occurred off of the Roadway",
25: "Toll Booth/Plaza Related",
26: "Prior Non-Recurring Incident",
27: "Backup Due to Prior Crash",
28: "Regular Congestion",
30: "Obstructed Crosswalks",
31: "Related to a Bus Stop",
40: "Traffic Controls Not Functioning Properly",
42: "Uncontrolled Intersection or Railroad Crossing",
99: "Unknown",
}
CRASHRF_NODE_ROWS = [
{"id": crashrf_id, "name": name}
for crashrf_id, name in CRASHRF_NODES.items()
]
LEGACY_1975_1981_MAP: dict[int, int] = {
24: 7,
40: 40,
41: 1,
42: 42,
43: 2,
44: 2,
47: 3,
48: 4,
49: 5,
50: 6,
51: 7,
99: 99,
}
DIRECT_1982_PLUS_MAP: dict[int, int] = {
1: 1,
2: 2,
3: 3,
4: 4,
5: 5,
6: 6,
7: 7,
10: 10,
12: 12,
13: 13,
14: 14,
15: 15,
16: 16,
17: 17,
18: 18,
19: 19,
20: 20,
21: 21,
22: 22,
23: 23,
24: 24,
25: 25,
26: 26,
27: 27,
28: 28,
30: 30,
31: 31,
40: 40,
42: 42,
99: 99,
999: 99,
}
MERGE_CRASHRF_NODES_QUERY = """
UNWIND $rows AS row
MERGE (c:CrashRF {id: row.id})
SET c.name = row.name
RETURN count(c) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (a:Accident {accident_id: row.accident_id})
MATCH (c:CrashRF {id: row.crashrf_id})
MERGE (c)-[r:CRASH_FACTOR_OF]->(a)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "crashrf_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS CrashRF-to-Accident relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure CrashRF constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def map_crashrf_code(year: int, code: int) -> int | None:
if code == 0:
return None
if 1975 <= year <= 1981:
return LEGACY_1975_1981_MAP.get(code)
return DIRECT_1982_PLUS_MAP.get(code)
def add_factor(
factors: dict[tuple[str, int], dict[str, Any]],
accident_id: str,
year: int,
crashrf_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (accident_id, crashrf_id)
factor = factors.setdefault(
key,
{
"accident_id": accident_id,
"crashrf_id": crashrf_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in factor["source_columns"]:
factor["source_columns"].append(source_column)
if source_code not in factor["source_codes"]:
factor["source_codes"].append(source_code)
def find_accident_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if 1975 <= year <= 2019:
csv_path = year_dir / "National" / "accident.csv"
if csv_path.exists():
files.append((year, csv_path))
return files
def find_crashrf_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if 2020 <= year <= 2022:
csv_path = year_dir / "National" / "crashrf.csv"
if csv_path.exists():
files.append((year, csv_path))
return files
def iter_accident_batches(year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in {"ST_CASE", "CF1", "CF2", "CF3"},
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
factors: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
accident_id = f"{year}_{identifier_part(record['ST_CASE'])}"
for column in ("CF1", "CF2", "CF3"):
code = int_code(record.get(column))
if code is None:
continue
crashrf_id = map_crashrf_code(year, code)
if crashrf_id is None:
continue
add_factor(factors, accident_id, year, crashrf_id, "ACCIDENT", column, code)
yield list(factors.values())
def iter_crashrf_batches(year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in {"ST_CASE", "CRASHRF"},
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
factors: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
code = int_code(record.get("CRASHRF"))
if code is None:
continue
crashrf_id = map_crashrf_code(year, code)
if crashrf_id is None:
continue
accident_id = f"{year}_{identifier_part(record['ST_CASE'])}"
add_factor(factors, accident_id, year, crashrf_id, "CRASHRF", "CRASHRF", code)
yield list(factors.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path in find_accident_files(base):
for rows in iter_accident_batches(year, csv_path, chunk_size):
yield year, "ACCIDENT", rows
for year, csv_path in find_crashrf_files(base):
for rows in iter_crashrf_batches(year, csv_path, chunk_size):
yield year, "CRASHRF", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_crashrf: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
total = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total += len(rows)
for row in rows:
rows_by_crashrf[int(row["crashrf_id"])] += 1
print(f"Expected CRASH_FACTOR_OF relationships: {total:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by CrashRF id : {dict(sorted(rows_by_crashrf.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT crashrf_id IF NOT EXISTS FOR (c:CrashRF) REQUIRE c.id IS UNIQUE",
":CrashRF(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT accident_id IF NOT EXISTS FOR (a:Accident) REQUIRE a.accident_id IS UNIQUE",
":Accident(accident_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} rows in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} rows in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_CRASHRF_NODES_QUERY, rows=CRASHRF_NODE_ROWS).consume()
print("Ensured CrashRF category nodes")
if args.ensure_constraints_only:
print(f"CrashRF constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"CrashRF edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:CRASH_FACTOR_OF]->() RETURN count(r) AS count").single()["count"]
print(f"CrashRF edge load finished UTC: {utc_now()}")
print(f"CrashRF edge load seconds : {load_elapsed:.3f}")
print(f"Total source rows processed : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"CRASH_FACTOR_OF rel count : {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Damage to Vehicle Relationship (426.056s)
Average edge import time per year: 6.281s across 1975-2011.
Average edge import time per year: 17.604s across 2012-2022.
python
#!/usr/bin/env python3
"""
Create FARS Damage-to-Vehicle relationships in Neo4j.
Sources:
* 1975-2011: vehicle-family IMPACT1/IMPACT2 columns
vehicle.csv, vehnit.csv, and parkwork.csv where present
* 2012-2022: damage.csv, one row per damaged area
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:Damage)-[:POSITION_OF]->(:Vehicle)
For 1980-1993, legacy IMPACT codes 15 and 16 indicate underride/override. They
are imported as Damage category nodes 15 and 16 and linked with POSITION_OF.
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
DAMAGE_NODES: dict[int, str] = {
0: "Non-Collision",
1: "Clock Point 1",
2: "Clock Point 2",
3: "Clock Point 3",
4: "Clock Point 4",
5: "Clock Point 5",
6: "Clock Point 6",
7: "Clock Point 7",
8: "Clock Point 8",
9: "Clock Point 9",
10: "Clock Point 10",
11: "Clock Point 11",
12: "Clock Point 12",
13: "Top",
14: "Undercarriage",
15: "Underride",
16: "Override",
18: "Set-in-Motion (Not a Clock Point)",
61: "Left",
62: "Left-Front Half",
63: "Left-Back Half",
81: "Right",
82: "Right-Front Half",
83: "Right-Back Half",
98: "Not Reported",
99: "Unknown",
}
DAMAGE_NODE_ROWS = [
{"id": damage_id, "name": name}
for damage_id, name in DAMAGE_NODES.items()
]
VALID_DAMAGE_IDS = set(DAMAGE_NODES)
MERGE_DAMAGE_NODES_QUERY = """
UNWIND $rows AS row
MERGE (d:Damage {id: row.id})
SET d.name = row.name
RETURN count(d) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
MATCH (d:Damage {id: row.damage_id})
MERGE (d)-[r:POSITION_OF]->(v)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "damage_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Damage-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Damage constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def canonical_column(name: Any, table: str) -> str:
column = normalize_column(name)
if table == "PARKWORK" and column.startswith("P"):
return column[1:]
return column
def normalize_columns(columns: Iterable[Any], table: str) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = canonical_column(column, table)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
return f"{year}_{identifier_part(st_case)}_{identifier_part(veh_no)}"
def valid_damage_code(code: int) -> bool:
return code in VALID_DAMAGE_IDS
def map_damage_code(year: int, code: int) -> int | None:
if 1980 <= year <= 1993 and code in {15, 16}:
return code
if code in {15, 16}:
return None
if valid_damage_code(code):
return code
return None
def add_damage(
damage_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
damage_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, damage_id)
row = damage_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"damage_id": damage_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_vehicle_family_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if not (1975 <= year <= 2011):
continue
national = year_dir / "National"
for table in ("vehicle", "vehnit", "parkwork"):
csv_path = national / f"{table}.csv"
if csv_path.exists():
files.append((year, table.upper(), csv_path))
return files
def find_damage_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if 2012 <= year <= 2022:
csv_path = year_dir / "National" / "damage.csv"
if csv_path.exists():
files.append((year, csv_path))
return files
def iter_vehicle_family_batches(
year: int,
table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "IMPACT1", "IMPACT2"}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: canonical_column(c, table) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns, table)
damage_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record["ST_CASE"], record["VEH_NO"])
for source_column in ("IMPACT1", "IMPACT2"):
code = int_code(record.get(source_column))
if code is None:
continue
damage_id = map_damage_code(year, code)
if damage_id is None:
continue
add_damage(damage_rows, vehicle_id, year, damage_id, table, source_column, code)
yield list(damage_rows.values())
def iter_damage_table_batches(
year: int,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
header = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
damage_column = "DAMAGE" if "DAMAGE" in columns else "MDAREAS"
wanted = {"ST_CASE", "VEH_NO", damage_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns, "DAMAGE")
damage_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
code = int_code(record.get(damage_column))
if code is None:
continue
damage_id = map_damage_code(year, code)
if damage_id is None:
continue
vehicle_id = vehicle_id_for(year, record["ST_CASE"], record["VEH_NO"])
add_damage(damage_rows, vehicle_id, year, damage_id, "DAMAGE", damage_column, code)
yield list(damage_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, table, csv_path in find_vehicle_family_files(base):
for damage_rows in iter_vehicle_family_batches(year, table, csv_path, chunk_size):
yield year, table, damage_rows
for year, csv_path in find_damage_files(base):
for damage_rows in iter_damage_table_batches(year, csv_path, chunk_size):
yield year, "DAMAGE", damage_rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_damage: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
total_damage_rows = 0
for year, source, damage_rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(damage_rows)
rows_by_source[source] += len(damage_rows)
total_damage_rows += len(damage_rows)
for row in damage_rows:
rows_by_damage[int(row["damage_id"])] += 1
print(f"Expected POSITION_OF relationships : {total_damage_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Damage id : {dict(sorted(rows_by_damage.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT damage_id IF NOT EXISTS FOR (d:Damage) REQUIRE d.id IS UNIQUE",
":Damage(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_damage_rows = 0
total_damage_matched = 0
current_year: int | None = None
year_damage_rows = 0
year_start = time.time()
for year, _, damage_rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(
f" Done {current_year}: {year_damage_rows:,} edges "
f"in {seconds_text(elapsed)} ({format_duration(elapsed)})"
)
current_year = year
year_damage_rows = 0
year_start = time.time()
if damage_rows:
record = session.run(MERGE_EDGE_QUERY, rows=damage_rows).single()
matched = int(record["matched"]) if record else 0
total_damage_rows += len(damage_rows)
total_damage_matched += matched
year_damage_rows += len(damage_rows)
if current_year is not None:
elapsed = time.time() - year_start
print(
f" Done {current_year}: {year_damage_rows:,} edges "
f"in {seconds_text(elapsed)} ({format_duration(elapsed)})"
)
return total_damage_rows, total_damage_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_DAMAGE_NODES_QUERY, rows=DAMAGE_NODE_ROWS).consume()
print("Ensured Damage category nodes")
if args.ensure_constraints_only:
print(f"Damage constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Damage edge load started UTC: {utc_now()}")
total_damage_rows, total_damage_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:POSITION_OF]->() RETURN count(r) AS count").single()["count"]
print(f"Damage edge load finished UTC: {utc_now()}")
print(f"Damage edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_damage_rows:,}")
print(f"Total relationships matched : {total_damage_matched:,}")
print(f"POSITION_OF relationship count: {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Distraction to Vehicle Relationship (28.560s)
Average edge import time per year: 1.527s across 2010-2017.
Average edge import time per year: 3.269s across 2018-2022.
python
#!/usr/bin/env python3
"""
Create FARS Distract-to-Vehicle relationships in Neo4j.
Sources:
* 2010-2019: distract.csv MDRDSTRD
* 2020-2022: distract.csv DRDISTRACT
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:Distract)-[:DISTRACTION_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
DISTRACT_NODES: dict[int, str] = {
1: "Looked but Did Not See",
3: "By Other Occupant(s)",
4: "By a Moving Object in Vehicle",
5: "While Talking or Listening to Mobile Phone",
6: "While Manipulating Mobile Phone",
7: "While Adjusting Audio or Climate Controls",
9: "While Using Other Component/Controls Integral to Vehicle",
10: "While Using or Reaching for Device/Object Brought Into Vehicle",
12: "Distracted by Outside Person, Object or Event",
13: "Eating or Drinking",
14: "Smoking Related",
15: "Other Mobile Phone Related",
16: "No Driver Present/Unknown if Driver Present",
17: "Distraction/Inattention",
18: "Distraction/Careless",
19: "Careless/Inattentive",
92: "Distraction (Distracted), Details Unknown",
93: "Inattention (Inattentive), Details Unknown",
96: "Not Reported",
97: "Lost in Thought/Daydreaming",
98: "Other Distraction",
99: "Reported as Unknown if Distracted",
}
DISTRACT_NODE_ROWS = [
{"id": distract_id, "name": name}
for distract_id, name in DISTRACT_NODES.items()
]
VALID_DISTRACT_IDS = set(DISTRACT_NODES)
MERGE_DISTRACT_NODES_QUERY = """
UNWIND $rows AS row
MERGE (d:Distract {id: row.id})
SET d.name = row.name
RETURN count(d) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
MATCH (d:Distract {id: row.distract_id})
MERGE (d)-[r:DISTRACTION_OF]->(v)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_column = row.source_column,
r.source_code = row.source_code,
r.created_from = "distract_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Distract-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Distract constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def find_source_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["distract.csv", f"DISTRACT_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "DRDISTRACT" in columns:
files.append((year, path, "DRDISTRACT"))
elif "MDRDSTRD" in columns:
files.append((year, path, "MDRDSTRD"))
return files
def add_distract_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
distract_id: int,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, distract_id)
edge_rows[key] = {
"vehicle_id": vehicle_id,
"distract_id": distract_id,
"DATA_YEAR": year,
"source_table": "DISTRACT",
"source_column": source_column,
"source_code": source_code,
}
def iter_source_batches(
year: int,
csv_path: Path,
source_column: str,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
distract_id = int_code(record.get(source_column))
if distract_id is None or distract_id not in VALID_DISTRACT_IDS:
continue
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
add_distract_edge(edge_rows, vehicle_id, year, distract_id, source_column, distract_id)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, list[dict[str, Any]]]]:
for year, csv_path, source_column in find_source_files(base):
for rows in iter_source_batches(year, csv_path, source_column, chunk_size):
yield year, rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_distract: Counter[int] = Counter()
total_rows = 0
for year, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_distract[int(row["distract_id"])] += 1
print(f"Expected DISTRACTION_OF relationships : {total_rows:,}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Distract id : {dict(sorted(rows_by_distract.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT distract_id IF NOT EXISTS FOR (d:Distract) REQUIRE d.id IS UNIQUE",
":Distract(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if rows:
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_DISTRACT_NODES_QUERY, rows=DISTRACT_NODE_ROWS).consume()
print("Ensured Distract category nodes")
if args.ensure_constraints_only:
print(f"Distract constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Distract edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:DISTRACTION_OF]->() RETURN count(r) AS count").single()["count"]
print(f"Distract edge load finished UTC : {utc_now()}")
print(f"Distract edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"DISTRACTION_OF relationship count: {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Driver Impairment to Vehicle Relationship (78.102s)
Average edge import time per year: 1.244s across 1975-2009.
Average edge import time per year: 2.657s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS DrImpair-to-Vehicle relationships in Neo4j.
Sources:
* 1975-2009: vehicle-family DR_CF1/DR_CF2/DR_CF3/DR_CF4 columns
vehicle.csv, and vehnit.csv where present
* 2010-2022: drimpair.csv DRIMPAIR
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:DrImpair)-[:IMPAIRMENT_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
DRIMPAIR_NODES: dict[int, str] = {
1: "Ill, Blackout",
2: "Asleep or Fatigued",
3: "Walking With a Cane or Crutches, etc.",
4: "Paraplegic or Restricted to Wheelchair",
5: "Impaired Due to Previous Injury",
6: "Deaf/Hard of Hearing",
7: "Blind/Low Vision",
8: "Emotional (Depressed, Angry, Disturbed, etc.)",
9: "Under the Influence of Alcohol, Drugs, or Medication",
10: "Physical Impairment - No Details",
95: "No Driver Present/Unknown if Driver Present",
96: "Other Physical Impairment",
98: "Not Reported",
99: "Unknown if Impaired",
}
DRIMPAIR_NODE_ROWS = [
{"id": drimpair_id, "name": name}
for drimpair_id, name in DRIMPAIR_NODES.items()
]
VALID_DRIMPAIR_IDS = set(DRIMPAIR_NODES)
MERGE_DRIMPAIR_NODES_QUERY = """
UNWIND $rows AS row
MERGE (d:DrImpair {id: row.id})
SET d.name = row.name
RETURN count(d) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
MATCH (d:DrImpair {id: row.drimpair_id})
MERGE (d)-[r:IMPAIRMENT_OF]->(v)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "drimpair_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS DrImpair-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure DrImpair constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def map_legacy_driver_factor(year: int, code: int) -> int | None:
if 1975 <= year <= 1981:
return {
2: 1, # Ill, blackout.
1: 2, # Drowsy/sleepy/asleep/fatigued.
3: 8, # Depression/emotional.
7: 10, # Physical impairments, no details.
}.get(code)
if 1982 <= year <= 2009:
mapping = {
2: 1, # Ill, passed out/blackout.
1: 2, # Drowsy/sleepy/asleep/fatigued.
3: 8, # Emotional.
7: 4, # Restricted to wheelchair.
9: 5, # Impaired due to previous injury.
11: 96, # Other physical impairment.
}
if 1982 <= year <= 1994:
mapping[8] = 4 # Paraplegic.
mapping[10] = 6 # Deaf.
if 2003 <= year <= 2009:
mapping[5] = 9 # Alcohol/drugs/medication.
return mapping.get(code)
return None
def add_drimpair_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
drimpair_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, drimpair_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"drimpair_id": drimpair_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_source_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit))
return files
def find_drimpair_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["drimpair.csv", f"DRIMPAIR_{year}.CSV"])
if path is not None:
files.append((year, path))
return files
def iter_legacy_batches(
year: int,
source_table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
header = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
dr_columns = [column for column in ("DR_CF1", "DR_CF2", "DR_CF3", "DR_CF4") if column in columns]
wanted = {"ST_CASE", "VEH_NO", *dr_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in dr_columns:
code = int_code(record.get(source_column))
if code is None:
continue
drimpair_id = map_legacy_driver_factor(year, code)
if drimpair_id is None:
continue
add_drimpair_edge(edge_rows, vehicle_id, year, drimpair_id, source_table, source_column, code)
yield list(edge_rows.values())
def iter_drimpair_batches(
year: int,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "DRIMPAIR"}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
code = int_code(record.get("DRIMPAIR"))
if code is None or code not in VALID_DRIMPAIR_IDS:
continue
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
add_drimpair_edge(edge_rows, vehicle_id, year, code, "DRIMPAIR", "DRIMPAIR", code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path in find_legacy_source_files(base):
for rows in iter_legacy_batches(year, source_table, csv_path, chunk_size):
yield year, source_table, rows
for year, csv_path in find_drimpair_files(base):
for rows in iter_drimpair_batches(year, csv_path, chunk_size):
yield year, "DRIMPAIR", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_drimpair: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_drimpair[int(row["drimpair_id"])] += 1
print(f"Expected IMPAIRMENT_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by DrImpair id : {dict(sorted(rows_by_drimpair.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT drimpair_id IF NOT EXISTS FOR (d:DrImpair) REQUIRE d.id IS UNIQUE",
":DrImpair(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if rows:
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_DRIMPAIR_NODES_QUERY, rows=DRIMPAIR_NODE_ROWS).consume()
print("Ensured DrImpair category nodes")
if args.ensure_constraints_only:
print(f"DrImpair constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"DrImpair edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:IMPAIRMENT_OF]->() RETURN count(r) AS count").single()["count"]
print(f"DrImpair edge load finished UTC : {utc_now()}")
print(f"DrImpair edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"IMPAIRMENT_OF relationship count: {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Driver Related Factor to Vehicle Relationship (238.912s)
Average edge import time per year: 6.345s across 1975-1981.
Average edge import time per year: 5.539s across 1982-2009.
Average edge import time per year: 2.989s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS DriverRF-to-Vehicle relationships in Neo4j.
Sources:
* 1975-2009: vehicle-family DR_CF1..DR_CF4 columns from vehicle.csv and
vehnit.csv where present
* 2010-2019: vehicle.csv DR_SF1..DR_SF4
* 2020-2022: driverrf.csv DRIVERRF
Some raw numeric codes changed meaning across years. This importer only maps a
code to the requested DriverRF node for years where the workbook shows the same
or compatible meaning.
Relationship:
(:DriverRF)-[:DRIVER_RF_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
DRIVERRF_NODES: dict[int, str] = {
4: "Reaction to or Failure to Take Drugs/Medication",
5: "Other Drugs (Marijuana, Cocaine, etc.)",
6: "Careless Driving, Inattentive Operation, Improper Driving, Driving Without Due Care",
8: "Road Rage/Aggressive Driving",
9: "Emergency Services Personnel",
10: "Looked but Did Not See",
12: "Mother of Dead Fetus/Mother of Infant Born Post Crash",
13: "Mentally Challenged",
14: "Failure to Take Drugs/Medication",
15: "Seat Back Not in Normal Position, Seat Back Reclined",
16: "Police or Law Enforcement Officer",
17: "Running off Road",
18: "Traveling on Prohibited Trafficways",
19: "Legally Driving on Suspended or Revoked License",
20: "Leaving Vehicle Unattended With Engine Running; Leaving Vehicle Unattended in Roadway",
21: "Overloading or Improper Loading of Vehicle With Passenger or Cargo",
22: "Towing or Pushing Vehicle Improperly",
23: "Failing to Dim Lights or to Have Lights on When Required",
24: "Operating Without Required Equipment",
25: "Creating Unlawful Noise or Using Equipment Prohibited by Law",
26: "Following Improperly",
27: "Improper or Erratic Lane Changing",
28: "Improper Lane Usage",
29: "Intentional Illegal Driving off the Roadway",
30: "Making Improper Entry to or Exit From Trafficway",
31: "Starting or Backing Improperly",
32: "Opening Vehicle Closure Into Moving Traffic or Vehicle Is in Motion",
33: "Passing Where Prohibited by Posted Signs, Pavement Markings, or School Bus Displaying Warning Not to Pass",
34: "Improper Passing Location",
35: "Passing With Insufficient Distance or Inadequate Visibility or Failing to Yield to Overtaking Vehicle",
36: "Operating the Vehicle in an Erratic, Reckless, Careless or Negligent Manner",
37: "Police Pursuing This Driver or Police Officer in Pursuit",
38: "Failure to Yield Right-of-Way",
39: "Failure to Obey Actual Traffic Signs, Traffic Control Devices or Traffic Officers, Failure to Observe Safety Zone Traffic Laws",
40: "Passing Through or Around Barrier",
41: "Failure to Observe Warnings or Instructions on Vehicle Displaying Them",
42: "Failure to Signal Intentions",
43: "Giving Wrong Signal",
44: "Driving Too Fast for Conditions or in Excess of Posted Speed Limit",
45: "Driving Less Than Posted Maximum",
46: "Operating at Erratic or Suddenly Changing Speeds",
47: "Making Right Turn From Left-Turn Lane or Making Left Turn From Right-Turn Lane",
48: "Making Improper Turn",
49: "Failure to Comply With Physical Restrictions of License",
50: "Driving Wrong Way on One-Way Trafficway",
51: "Driving on Wrong Side of Two-way Trafficway (Intentionally or Unintentionally)",
52: "Operator Inexperience",
53: "Unfamiliar With Roadway",
54: "Stopping in Roadway (Vehicle Not Abandoned)",
55: "Improper Management of Vehicle Controls",
56: "Object Interference With Vehicle Controls",
57: "Driving With Tire-Related Problems",
58: "Over Correcting",
59: "Getting off/out of a Vehicle",
60: "Alcohol and/or Drug Test Refused",
73: "Driver Has Not Complied With Learners Permit or Intermediate Driver License Restrictions (GDL Restrictions)",
74: "Driver Has Not Complied With Physical or Other Imposed Restrictions",
77: "Severe Crosswind",
78: "Wind From Passing Truck",
79: "Slippery or Loose Surface",
80: "Tire Blow-Out or Flat",
81: "Debris or Objects in Road",
82: "Ruts, Holes, Bumps in Road",
83: "Live Animals in Road",
84: "Vehicle in Road",
85: "Phantom Vehicle",
86: "Pedestrian, Pedalcyclist, or Other Non-Motorist in Road",
87: "Ice, Water, Snow, Slush, Sand, Dirt, Oil, Wet Leaves on Road",
88: "Trailer Fishtailing or Swaying",
89: "Driver has a Driving Record or Driver's License From More Than One State",
90: "Hit-and-Run Vehicle Driver",
91: "Non-Traffic Violation Charged (Manslaughter, Homicide or Other Assault Offense Committed Without Malice)",
92: "Other Non-Moving Traffic Violation",
93: "Cellular Telephone",
94: "Emergency Medical Service Personnel",
95: "Fire Personnel",
96: "Tow Operator",
97: "Transportation (i.e., Maintenance Workers, Safety Service Patrol Operators, etc.)",
98: "Head-Up Display (1991-2009)",
999: "Reported as Unknown",
}
DRIVERRF_NODE_ROWS = [{"id": code, "name": name} for code, name in DRIVERRF_NODES.items()]
MERGE_DRIVERRF_NODES_QUERY = """
UNWIND $rows AS row
MERGE (rf:DriverRF {id: row.id})
SET rf.name = row.name
RETURN count(rf) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (veh:Vehicle {vehicle_id: row.vehicle_id})
MATCH (rf:DriverRF {id: row.driverrf_id})
MERGE (rf)-[r:DRIVER_RF_OF]->(veh)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "driverrf_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS DriverRF-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure DriverRF constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def is_valid_driverrf_for_year(year: int, code: int) -> bool:
if code not in DRIVERRF_NODES:
return False
if code == 5:
return 1975 <= year <= 1994
if code == 8:
return year >= 2004
if code == 9:
return year == 2018
if code == 10:
return year >= 2018
if code == 12:
return year >= 1982
if code == 13:
return year >= 1995
if code == 14:
return 1995 <= year <= 2004
if code in {15, 16}:
return year >= 2002
if code == 17:
return 2000 <= year <= 2003
if code == 18:
return year >= 1995
if code == 19:
return year >= 1982
if code == 25:
return year <= 2009
if code == 37:
return year >= 1978
if code == 43:
return year <= 1981
if code == 44:
return year <= 2008
if code == 45:
return year >= 1982
if code == 46:
return year <= 1994
if code == 49:
return year <= 2004
if code in {55, 56, 57, 60}:
return year >= 2017
if code == 59:
return year >= 1982
if code in {73, 74}:
return year >= 2004
if code in {77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88}:
return year >= 1982
if code == 89:
return year >= 2010
if code == 90:
return 1982 <= year <= 2009
if code == 91:
return 1986 <= year <= 2020
if code == 92:
return 1986 <= year <= 2011
if code in {93, 98}:
return 1991 <= year <= 2009
if code in {94, 95, 96, 97}:
return year >= 2019
if code == 999:
return year >= 2020
return True
def add_driverrf_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
driverrf_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, driverrf_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"driverrf_id": driverrf_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_source_files(base: Path) -> list[tuple[int, str, Path, tuple[str, ...]]]:
files: list[tuple[int, str, Path, tuple[str, ...]]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
columns = ("DR_CF1", "DR_CF2", "DR_CF3", "DR_CF4")
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle, columns))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit, columns))
return files
def find_driversf_source_files(base: Path) -> list[tuple[int, str, Path, tuple[str, ...]]]:
files: list[tuple[int, str, Path, tuple[str, ...]]] = []
for year in range(2010, 2020):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle, ("DR_SF1", "DR_SF2", "DR_SF3", "DR_SF4")))
return files
def find_driverrf_source_files(base: Path) -> list[tuple[int, str, Path, tuple[str, ...]]]:
files: list[tuple[int, str, Path, tuple[str, ...]]] = []
for year in range(2020, 2023):
national = base / str(year) / "National"
path = find_file(national, ["driverrf.csv", f"DRIVERRF_{year}.CSV"])
if path is not None:
files.append((year, "DRIVERRF", path, ("DRIVERRF",)))
return files
def iter_source_batches(
year: int,
source_table: str,
csv_path: Path,
source_columns: tuple[str, ...],
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
header = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
available = {normalize_column(column) for column in header.columns}
active_columns = tuple(column for column in source_columns if column in available)
if not active_columns:
return
wanted = {"ST_CASE", "VEH_NO", *active_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in active_columns:
code = int_code(record.get(source_column))
if code is None or not is_valid_driverrf_for_year(year, code):
continue
add_driverrf_edge(edge_rows, vehicle_id, year, code, source_table, source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path, source_columns in find_legacy_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, source_columns, chunk_size):
yield year, source_table, rows
for year, source_table, csv_path, source_columns in find_driversf_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, source_columns, chunk_size):
yield year, source_table, rows
for year, source_table, csv_path, source_columns in find_driverrf_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, source_columns, chunk_size):
yield year, source_table, rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_driverrf: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_driverrf[int(row["driverrf_id"])] += 1
print(f"Expected DRIVER_RF_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by DriverRF id : {dict(sorted(rows_by_driverrf.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT driverrf_id IF NOT EXISTS FOR (rf:DriverRF) REQUIRE rf.id IS UNIQUE",
":DriverRF(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_DRIVERRF_NODES_QUERY, rows=DRIVERRF_NODE_ROWS).single()
print(f"Ensured DriverRF nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal DRIVER_RF_OF rows sent: {rows_sent:,}")
print(f"Total DRIVER_RF_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Drug Event to Person Relationship (228.852s)
This import is performed as one total batch rather than by year, so no yearly average time is available.
python
#!/usr/bin/env python3
"""
Create FARS DrugEvent nodes from historical person drug columns and drugs.csv.
Sources:
* 1991-1992: PERSON DRUGTEST / DRUG_RES, one event slot per person row.
* 1993-2017: PERSON DRUGTST1..3 / DRUGRES1..3, up to three event slots.
* 2018-2022: drugs.csv DRUGSPEC / DRUGRES, one event per drugs.csv row.
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. DrugEvent
nodes also keep person_id as a property and are linked with:
(:DrugEvent)-[:DRUG_EVENT_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Iterator
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
MERGE_DRUG_EVENT_QUERY = """
UNWIND $rows AS row
MERGE (d:DrugEvent {drug_event_id: row.drug_event_id})
SET d += row
WITH d, row
OPTIONAL MATCH (p:Person {person_id: row.person_id})
FOREACH (_ IN CASE WHEN p IS NULL THEN [] ELSE [1] END |
MERGE (d)-[:DRUG_EVENT_OF]->(p)
)
RETURN count(d) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS DrugEvent nodes in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure constraints, then exit without importing nodes.",
)
return parser.parse_args()
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def clean_text(value: Any) -> str | None:
if value is None or pd.isna(value):
return None
text = str(value).replace("\x1a", "").strip()
if len(text) >= 2 and text[0] == '"' and text[-1] == '"':
text = text[1:-1].strip()
text = " ".join(text.split())
return text or None
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def has_drug_event(drugtest_code: int | None, drugres_code: int | None) -> bool:
if drugtest_code not in (None, 0):
return True
return drugres_code not in (None, 0, 1)
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def read_csv(path: Path) -> pd.DataFrame:
for encoding in ("utf-8", "latin-1"):
try:
df = pd.read_csv(path, encoding=encoding, low_memory=False)
df.columns = normalize_columns(df.columns)
return df
except UnicodeDecodeError:
continue
raise RuntimeError(f"Could not decode {path}")
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def find_person_file(base: Path, year: int) -> Path | None:
national = base / str(year) / "National"
return find_file(national, [f"PERSON_{year}.CSV", "person.csv", "PERSON.CSV", "Person.csv"])
def find_drugs_file(base: Path, year: int) -> Path | None:
national = base / str(year) / "National"
return find_file(national, [f"DRUGS_{year}.CSV", "drugs.csv", "DRUGS.CSV", "Drugs.csv"])
def base_row(
year: int,
person_id: str,
st_case: Any,
veh_no: Any,
per_no: Any,
source_table: str,
source_slot: str,
source_row_number: int,
drugtest_code: int | None,
drugres_code: int | None,
drugtest_name: str | None = None,
drugres_name: str | None = None,
) -> dict[str, Any]:
drug_event_id = f"{person_id}_DRUG_{source_slot}"
return {
"drug_event_id": drug_event_id,
"person_id": person_id,
"DATA_YEAR": year,
"ST_CASE": person_id.rsplit("_", 2)[0],
"VEH_NO": identifier_part(veh_no),
"PER_NO": identifier_part(per_no),
"source_table": source_table,
"source_slot": source_slot,
"source_row_number": source_row_number,
"drugtest_code": drugtest_code,
"drugres_code": drugres_code,
"drugtest_name": drugtest_name,
"drugres_name": drugres_name,
"created_from": "drug_event_import",
}
def iter_person_drug_events(base: Path, year: int) -> Iterator[dict[str, Any]]:
path = find_person_file(base, year)
if path is None:
print(f" WARNING {year}: no PERSON file found")
return
df = read_csv(path)
required = {"ST_CASE", "VEH_NO", "PER_NO"}
missing = sorted(required - set(df.columns))
if missing:
print(f" WARNING {year}: {path.name} missing {missing}; skipping")
return
if year <= 1992:
pairs = [("1", "DRUGTEST", "DRUG_RES", None, None)]
else:
pairs = [
("1", "DRUGTST1", "DRUGRES1", "DRUGTST1NAME", "DRUGRES1NAME"),
("2", "DRUGTST2", "DRUGRES2", "DRUGTST2NAME", "DRUGRES2NAME"),
("3", "DRUGTST3", "DRUGRES3", "DRUGTST3NAME", "DRUGRES3NAME"),
]
for row_number, rec in enumerate(df.itertuples(index=False), start=1):
values = rec._asdict()
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
for slot, test_col, res_col, test_name_col, res_name_col in pairs:
if test_col not in df.columns or res_col not in df.columns:
continue
drugtest_code = int_code(values[test_col])
drugres_code = int_code(values[res_col])
if not has_drug_event(drugtest_code, drugres_code):
continue
yield base_row(
year=year,
person_id=person_id,
st_case=values["ST_CASE"],
veh_no=values["VEH_NO"],
per_no=values["PER_NO"],
source_table="PERSON",
source_slot=slot,
source_row_number=row_number,
drugtest_code=drugtest_code,
drugres_code=drugres_code,
drugtest_name=clean_text(values.get(test_name_col)) if test_name_col else None,
drugres_name=clean_text(values.get(res_name_col)) if res_name_col else None,
)
def iter_drugs_table_events(base: Path, year: int) -> Iterator[dict[str, Any]]:
path = find_drugs_file(base, year)
if path is None:
print(f" WARNING {year}: no drugs.csv file found")
return
df = read_csv(path)
required = {"ST_CASE", "VEH_NO", "PER_NO", "DRUGSPEC", "DRUGRES"}
missing = sorted(required - set(df.columns))
if missing:
print(f" WARNING {year}: {path.name} missing {missing}; skipping")
return
person_sequence: Counter[str] = Counter()
for row_number, rec in enumerate(df.itertuples(index=False), start=1):
values = rec._asdict()
drugtest_code = int_code(values["DRUGSPEC"])
drugres_code = int_code(values["DRUGRES"])
if not has_drug_event(drugtest_code, drugres_code):
continue
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
person_sequence[person_id] += 1
slot = str(person_sequence[person_id])
yield base_row(
year=year,
person_id=person_id,
st_case=values["ST_CASE"],
veh_no=values["VEH_NO"],
per_no=values["PER_NO"],
source_table="DRUGS",
source_slot=slot,
source_row_number=row_number,
drugtest_code=drugtest_code,
drugres_code=drugres_code,
drugtest_name=clean_text(values.get("DRUGSPECNAME")),
drugres_name=clean_text(values.get("DRUGRESNAME")),
)
def iter_all_events(base: Path) -> Iterator[dict[str, Any]]:
for year in range(1991, 2018):
yield from iter_person_drug_events(base, year)
for year in range(2018, 2023):
yield from iter_drugs_table_events(base, year)
def dry_run(base: Path) -> None:
total = 0
counts_by_year: Counter[int] = Counter()
counts_by_source: Counter[str] = Counter()
combo_counts: Counter[tuple[int | None, int | None]] = Counter()
start = time.time()
for row in iter_all_events(base):
total += 1
counts_by_year[row["DATA_YEAR"]] += 1
counts_by_source[row["source_table"]] += 1
combo_counts[(row["drugtest_code"], row["drugres_code"])] += 1
print(f"Dry run DrugEvent nodes prepared: {total}")
print(f"Counts by source: {dict(sorted(counts_by_source.items()))}")
print(f"Counts by year: {dict(sorted(counts_by_year.items()))}")
print(f"Distinct drugtest/drugres combinations: {len(combo_counts)}")
print("Top 20 combinations:")
for (test_code, res_code), count in combo_counts.most_common(20):
print(f" drugtest={test_code}, drugres={res_code}: {count}")
print(f"Dry-run elapsed seconds: {time.time() - start:.3f}")
def ensure_constraints(session: Any) -> None:
session.run(
"CREATE CONSTRAINT drugevent_id_unique IF NOT EXISTS "
"FOR (d:DrugEvent) REQUIRE d.drug_event_id IS UNIQUE"
).consume()
session.run(
"CREATE CONSTRAINT person_person_id_unique IF NOT EXISTS "
"FOR (p:Person) REQUIRE p.person_id IS UNIQUE"
).consume()
def import_events(session: Any, base: Path, batch_size: int) -> int:
total = 0
batch: list[dict[str, Any]] = []
year_counts: Counter[int] = Counter()
start = time.time()
for row in iter_all_events(base):
batch.append(row)
year_counts[row["DATA_YEAR"]] += 1
if len(batch) >= batch_size:
result = session.run(MERGE_DRUG_EVENT_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported batch: total={total}")
batch = []
if batch:
result = session.run(MERGE_DRUG_EVENT_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported final batch: total={total}")
print(f"Counts by year: {dict(sorted(year_counts.items()))}")
print(f"Import loop elapsed seconds: {time.time() - start:.3f}")
return total
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started at {utc_now()}")
print(f"Base: {args.base}")
print(f"Database: {args.database}")
if args.dry_run:
dry_run(args.base)
print(f"Finished at {utc_now()}")
print(f"Total elapsed seconds: {time.time() - total_start:.3f}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
if args.ensure_constraints_only:
print("Constraints ensured; no DrugEvent nodes imported.")
return
imported = import_events(session, args.base, args.batch_size)
print(f"Total DrugEvent nodes matched/created: {imported}")
finally:
driver.close()
total_elapsed = time.time() - total_start
print(f"Finished at {utc_now()}")
print(f"Total elapsed: {format_duration(total_elapsed)}")
print(f"Total elapsed seconds: {total_elapsed:.3f}")
if __name__ == "__main__":
main()
Drug Test Result to Drug Event Relationship (192.319s)
This import is performed as one total batch rather than by year, so no yearly average time is available.
python
#!/usr/bin/env python3
"""
Create Drug nodes and Drug-to-DrugEvent result relationships.
Rules:
* 2018-2022: create Drug nodes from drugres_code_name_mapping_2018_2022.csv
and link by DRUGRES code.
* 2015-2017: lowercase DrugEvent.drugres_name, then link by name to the
existing 2018-2022 Drug nodes. If no name match exists, link to/create a
historical Drug node with id = 19932017<drugres_code>.
* 1993-2014: no name exists in PERSON, so link by historical Drug node id
19932017<drugres_code>. The node name comes from the 2015-2017 mapping
when available; otherwise it is "N/A".
Relationship:
(:Drug)-[:RESULT_OF]->(:DrugEvent)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Iterator
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
MODERN_MAPPING = "drugres_code_name_mapping_2018_2022.csv"
LEGACY_MAPPING = "drugres_code_name_mapping_2015_2017.csv"
LEGACY_PREFIX = "19932017"
MERGE_DRUG_NODES_QUERY = """
UNWIND $rows AS row
MERGE (d:Drug {id: row.id, name: row.name})
SET d.code = row.code,
d.name_key = row.name_key,
d.source_mapping = row.source_mapping,
d.created_from = "drug_result_edges_import"
RETURN count(d) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (d:Drug {id: row.drug_id, name: row.drug_name})
MATCH (e:DrugEvent {drug_event_id: row.drug_event_id})
MERGE (d)-[r:RESULT_OF]->(e)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_slot = row.source_slot,
r.source_drugres_code = row.source_drugres_code,
r.match_method = row.match_method,
r.created_from = "drug_result_edges_import"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create Drug result nodes and RESULT_OF relationships.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument("--ensure-nodes-only", action="store_true", help="Only create/update Drug nodes.")
return parser.parse_args()
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def clean_name(value: Any) -> str | None:
if value is None or pd.isna(value):
return None
text = str(value).replace("\x1a", "").replace('"', "").replace("“", "").replace("”", "").strip()
text = " ".join(text.split()).lower()
return text or None
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def has_drug_event(drugtest_code: int | None, drugres_code: int | None) -> bool:
if drugtest_code not in (None, 0):
return True
return drugres_code not in (None, 0, 1)
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def read_csv(path: Path) -> pd.DataFrame:
for encoding in ("utf-8", "latin-1"):
try:
df = pd.read_csv(path, encoding=encoding, low_memory=False)
df.columns = normalize_columns(df.columns)
return df
except UnicodeDecodeError:
continue
raise RuntimeError(f"Could not decode {path}")
def read_mapping(path: Path) -> pd.DataFrame:
df = pd.read_csv(path, escapechar="\\")
df.columns = [str(c).strip().lower() for c in df.columns]
if not {"code", "name"}.issubset(df.columns):
raise ValueError(f"{path} must contain code,name columns")
df["code"] = pd.to_numeric(df["code"], errors="coerce").astype("Int64")
df["name"] = df["name"].map(clean_name)
df = df.dropna(subset=["code", "name"])
df["code"] = df["code"].astype(int)
return df[["code", "name"]]
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def find_person_file(base: Path, year: int) -> Path | None:
national = base / str(year) / "National"
return find_file(national, [f"PERSON_{year}.CSV", "person.csv", "PERSON.CSV", "Person.csv"])
def find_drugs_file(base: Path, year: int) -> Path | None:
national = base / str(year) / "National"
return find_file(national, [f"DRUGS_{year}.CSV", "drugs.csv", "DRUGS.CSV", "Drugs.csv"])
def legacy_id(code: int) -> int:
return int(f"{LEGACY_PREFIX}{code}")
def drug_node(drug_id: int | str, code: int | None, name: str, source_mapping: str) -> dict[str, Any]:
return {
"id": drug_id,
"code": code,
"name": name,
"name_key": clean_name(name) if name != "N/A" else "n/a",
"source_mapping": source_mapping,
}
def load_modern_nodes(base: Path) -> tuple[
list[dict[str, Any]],
dict[int, dict[str, Any]],
dict[str, list[dict[str, Any]]],
]:
mapping = read_mapping(base / MODERN_MAPPING)
seen: Counter[tuple[int, str]] = Counter()
rows: list[dict[str, Any]] = []
by_code: dict[int, dict[str, Any]] = {}
by_name: dict[str, list[dict[str, Any]]] = defaultdict(list)
for record in mapping.itertuples(index=False):
code = int(record.code)
name = str(record.name)
seen[(code, name)] += 1
drug_id: int | str = code if seen[(code, name)] == 1 else f"{code}_{seen[(code, name)]}"
row = drug_node(drug_id, code, name, "drugres_code_name_mapping_2018_2022")
rows.append(row)
by_code.setdefault(code, row)
by_name[name].append(row)
return rows, by_code, by_name
def load_legacy_name_by_code(base: Path) -> dict[int, str]:
mapping = read_mapping(base / LEGACY_MAPPING)
return {int(row.code): str(row.name) for row in mapping.itertuples(index=False)}
def base_edge_row(
drug: dict[str, Any],
year: int,
person_id: str,
source_table: str,
source_slot: str,
source_drugres_code: int | None,
match_method: str,
) -> dict[str, Any]:
return {
"drug_id": drug["id"],
"drug_name": drug["name"],
"drug_event_id": f"{person_id}_DRUG_{source_slot}",
"DATA_YEAR": year,
"source_table": source_table,
"source_slot": source_slot,
"source_drugres_code": source_drugres_code,
"match_method": match_method,
}
def iter_2018_2022_edges(
base: Path,
by_code: dict[int, dict[str, Any]],
) -> Iterator[tuple[dict[str, Any] | None, str]]:
for year in range(2018, 2023):
path = find_drugs_file(base, year)
if path is None:
print(f" WARNING {year}: no drugs.csv found")
continue
df = read_csv(path)
person_sequence: Counter[str] = Counter()
for rec in df.itertuples(index=False):
values = rec._asdict()
drugtest_code = int_code(values["DRUGSPEC"])
drugres_code = int_code(values["DRUGRES"])
if not has_drug_event(drugtest_code, drugres_code):
continue
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
person_sequence[person_id] += 1
source_slot = str(person_sequence[person_id])
drug = by_code.get(drugres_code) if drugres_code is not None else None
if drug is None:
yield None, "unmatched_code_2018_2022"
continue
yield base_edge_row(drug, year, person_id, "DRUGS", source_slot, drugres_code, "code_2018_2022"), "code_2018_2022"
def iter_2015_2017_edges(
base: Path,
by_name: dict[str, list[dict[str, Any]]],
legacy_nodes: dict[int, dict[str, Any]],
) -> Iterator[tuple[dict[str, Any] | None, str]]:
for year in range(2015, 2018):
path = find_person_file(base, year)
if path is None:
print(f" WARNING {year}: no PERSON file found")
continue
df = read_csv(path)
for rec in df.itertuples(index=False):
values = rec._asdict()
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
for slot in ("1", "2", "3"):
test_col = f"DRUGTST{slot}"
res_col = f"DRUGRES{slot}"
name_col = f"DRUGRES{slot}NAME"
drugtest_code = int_code(values[test_col])
drugres_code = int_code(values[res_col])
if not has_drug_event(drugtest_code, drugres_code):
continue
name = clean_name(values.get(name_col))
if name and name in by_name:
for drug in by_name[name]:
yield base_edge_row(drug, year, person_id, "PERSON", slot, drugres_code, "name_2015_2017"), "name_2015_2017"
continue
if drugres_code is None:
yield None, "unmatched_no_code_2015_2017"
continue
lname = name or "N/A"
drug = legacy_nodes.setdefault(
drugres_code,
drug_node(legacy_id(drugres_code), drugres_code, lname, "legacy_1993_2017_from_2015_2017_unmatched"),
)
yield base_edge_row(drug, year, person_id, "PERSON", slot, drugres_code, "legacy_created_2015_2017"), "legacy_created_2015_2017"
def iter_1993_2014_edges(
base: Path,
legacy_name_by_code: dict[int, str],
legacy_nodes: dict[int, dict[str, Any]],
) -> Iterator[tuple[dict[str, Any] | None, str]]:
for year in range(1993, 2015):
path = find_person_file(base, year)
if path is None:
print(f" WARNING {year}: no PERSON file found")
continue
df = read_csv(path)
for rec in df.itertuples(index=False):
values = rec._asdict()
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
for slot in ("1", "2", "3"):
test_col = f"DRUGTST{slot}"
res_col = f"DRUGRES{slot}"
drugtest_code = int_code(values[test_col])
drugres_code = int_code(values[res_col])
if not has_drug_event(drugtest_code, drugres_code):
continue
if drugres_code is None:
yield None, "unmatched_no_code_1993_2014"
continue
if drugres_code in legacy_nodes:
drug = legacy_nodes[drugres_code]
method = "legacy_id_1993_2014"
else:
name = legacy_name_by_code.get(drugres_code, "N/A")
source = "legacy_1993_2017_mapping" if drugres_code in legacy_name_by_code else "legacy_1993_2017_na"
drug = legacy_nodes.setdefault(drugres_code, drug_node(legacy_id(drugres_code), drugres_code, name, source))
method = "legacy_id_1993_2014" if name != "N/A" else "legacy_id_created_na_1993_2014"
yield base_edge_row(drug, year, person_id, "PERSON", slot, drugres_code, method), method
def collect_legacy_nodes(
base: Path,
by_name: dict[str, list[dict[str, Any]]],
legacy_name_by_code: dict[int, str],
) -> dict[int, dict[str, Any]]:
legacy_nodes: dict[int, dict[str, Any]] = {}
for _row, _method in iter_2015_2017_edges(base, by_name, legacy_nodes):
pass
for _row, _method in iter_1993_2014_edges(base, legacy_name_by_code, legacy_nodes):
pass
return legacy_nodes
def iter_all_edges(
base: Path,
by_code: dict[int, dict[str, Any]],
by_name: dict[str, list[dict[str, Any]]],
legacy_name_by_code: dict[int, str],
legacy_nodes: dict[int, dict[str, Any]],
) -> Iterator[tuple[dict[str, Any] | None, str]]:
yield from iter_1993_2014_edges(base, legacy_name_by_code, legacy_nodes)
yield from iter_2015_2017_edges(base, by_name, legacy_nodes)
yield from iter_2018_2022_edges(base, by_code)
def dry_run(base: Path) -> None:
modern_nodes, by_code, by_name = load_modern_nodes(base)
legacy_name_by_code = load_legacy_name_by_code(base)
legacy_nodes = collect_legacy_nodes(base, by_name, legacy_name_by_code)
total_edges = 0
method_counts: Counter[str] = Counter()
year_counts: Counter[int] = Counter()
unmatched_counts: Counter[str] = Counter()
start = time.time()
for row, method in iter_all_edges(base, by_code, by_name, legacy_name_by_code, legacy_nodes):
method_counts[method] += 1
if row is None:
unmatched_counts[method] += 1
continue
total_edges += 1
year_counts[row["DATA_YEAR"]] += 1
print(f"Modern Drug nodes from 2018-2022 mapping: {len(modern_nodes)}")
print(f"Historical Drug nodes with id {LEGACY_PREFIX}<code>: {len(legacy_nodes)}")
print(f"Dry run RESULT_OF edges prepared: {total_edges}")
print(f"Counts by year: {dict(sorted(year_counts.items()))}")
print(f"Counts by match method: {dict(sorted(method_counts.items()))}")
print(f"Unmatched counts: {dict(sorted(unmatched_counts.items()))}")
print(f"Dry-run elapsed seconds: {time.time() - start:.3f}")
def normalize_drug_event_names(session: Any) -> None:
result = session.run(
"""
MATCH (e:DrugEvent)
WHERE e.drugres_name IS NOT NULL
SET e.drugres_name = toLower(e.drugres_name)
RETURN count(e) AS updated
"""
).single()
print(f"Lowercased DrugEvent.drugres_name values: {result['updated'] if result else 0}")
def ensure_indexes_and_nodes(session: Any, drug_nodes: list[dict[str, Any]]) -> None:
session.run("CREATE INDEX drug_id_name IF NOT EXISTS FOR (d:Drug) ON (d.id, d.name)").consume()
session.run("CREATE INDEX drug_code IF NOT EXISTS FOR (d:Drug) ON (d.code)").consume()
session.run("CREATE INDEX drug_name_key IF NOT EXISTS FOR (d:Drug) ON (d.name_key)").consume()
session.run(
"CREATE CONSTRAINT drugevent_id_unique IF NOT EXISTS "
"FOR (e:DrugEvent) REQUIRE e.drug_event_id IS UNIQUE"
).consume()
total = 0
for i in range(0, len(drug_nodes), DEFAULT_BATCH_SIZE):
batch = drug_nodes[i : i + DEFAULT_BATCH_SIZE]
result = session.run(MERGE_DRUG_NODES_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f"Ensured Drug nodes: {total}")
def import_edges(
session: Any,
base: Path,
by_code: dict[int, dict[str, Any]],
by_name: dict[str, list[dict[str, Any]]],
legacy_name_by_code: dict[int, str],
legacy_nodes: dict[int, dict[str, Any]],
batch_size: int,
) -> int:
total = 0
batch: list[dict[str, Any]] = []
method_counts: Counter[str] = Counter()
unmatched_counts: Counter[str] = Counter()
start = time.time()
for row, method in iter_all_edges(base, by_code, by_name, legacy_name_by_code, legacy_nodes):
method_counts[method] += 1
if row is None:
unmatched_counts[method] += 1
continue
batch.append(row)
if len(batch) >= batch_size:
result = session.run(MERGE_EDGE_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported batch: total={total}")
batch = []
if batch:
result = session.run(MERGE_EDGE_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported final batch: total={total}")
print(f"Counts by match method: {dict(sorted(method_counts.items()))}")
print(f"Unmatched counts: {dict(sorted(unmatched_counts.items()))}")
print(f"Import loop elapsed seconds: {time.time() - start:.3f}")
return total
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started at {utc_now()}")
print(f"Base: {args.base}")
print(f"Database: {args.database}")
if args.dry_run:
dry_run(args.base)
print(f"Finished at {utc_now()}")
print(f"Total elapsed seconds: {time.time() - total_start:.3f}")
return
from neo4j import GraphDatabase
modern_nodes, by_code, by_name = load_modern_nodes(args.base)
legacy_name_by_code = load_legacy_name_by_code(args.base)
legacy_nodes = collect_legacy_nodes(args.base, by_name, legacy_name_by_code)
all_drug_nodes = modern_nodes + list(legacy_nodes.values())
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
normalize_drug_event_names(session)
ensure_indexes_and_nodes(session, all_drug_nodes)
if args.ensure_nodes_only:
print("Drug nodes ensured; no RESULT_OF edges imported.")
return
imported = import_edges(session, args.base, by_code, by_name, legacy_name_by_code, legacy_nodes, args.batch_size)
print(f"Total RESULT_OF edges matched/created: {imported}")
finally:
driver.close()
total_elapsed = time.time() - total_start
print(f"Finished at {utc_now()}")
print(f"Total elapsed: {format_duration(total_elapsed)}")
print(f"Total elapsed seconds: {total_elapsed:.3f}")
if __name__ == "__main__":
main()
Drug Test to Drug Event Relationship (227.472s)
This import is performed as one total batch rather than by year, so no yearly average time is available.
python
#!/usr/bin/env python3
"""
Create FARS DrugTest-to-DrugEvent relationships.
Sources:
* 1991-1992: PERSON DRUGTEST / DRUG_RES
* 1993-2017: PERSON DRUGTST1..3 / DRUGRES1..3
* 2018-2022: drugs.csv DRUGSPEC / DRUGRES
Relationship:
(:DrugTest)-[:DRUG_TEST_OF]->(:DrugEvent)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Iterator
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
DRUGTEST_NODES: dict[int, str] = {
0: "Test Not Given",
1: "Whole Blood",
2: "Urine",
11: "Blood Plasma/Serum",
12: "Blood Clot",
13: "Oral Fluids",
14: "Vitreous",
15: "Liver",
96: "Not Reported",
97: "Unknown Specimen",
98: "Other Specimen",
99: "Reported as Unknown if Tested",
}
DRUGTEST_NODE_ROWS = [{"id": code, "name": name} for code, name in DRUGTEST_NODES.items()]
MERGE_DRUGTEST_NODES_QUERY = """
UNWIND $rows AS row
MERGE (t:DrugTest {id: row.id})
SET t.name = row.name
RETURN count(t) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (t:DrugTest {id: row.drugtest_id})
MATCH (d:DrugEvent {drug_event_id: row.drug_event_id})
MERGE (t)-[r:DRUG_TEST_OF]->(d)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_slot = row.source_slot,
r.source_drugtest_code = row.source_drugtest_code,
r.AND_FLAG = row.AND_FLAG,
r.OR_FLAG = row.OR_FLAG,
r.created_from = "drugtest_edges_import"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS DrugTest-to-DrugEvent relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure constraints and DrugTest nodes, then exit without importing relationships.",
)
return parser.parse_args()
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def has_drug_event(drugtest_code: int | None, drugres_code: int | None) -> bool:
if drugtest_code not in (None, 0):
return True
return drugres_code not in (None, 0, 1)
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def read_csv(path: Path) -> pd.DataFrame:
for encoding in ("utf-8", "latin-1"):
try:
df = pd.read_csv(path, encoding=encoding, low_memory=False)
df.columns = normalize_columns(df.columns)
return df
except UnicodeDecodeError:
continue
raise RuntimeError(f"Could not decode {path}")
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def find_person_file(base: Path, year: int) -> Path | None:
national = base / str(year) / "National"
return find_file(national, [f"PERSON_{year}.CSV", "person.csv", "PERSON.CSV", "Person.csv"])
def find_drugs_file(base: Path, year: int) -> Path | None:
national = base / str(year) / "National"
return find_file(national, [f"DRUGS_{year}.CSV", "drugs.csv", "DRUGS.CSV", "Drugs.csv"])
def drugtest_edges_for(year: int, code: int | None) -> list[dict[str, int | None]]:
if code is None:
return []
if 1991 <= year <= 2017:
if code in {0, 1, 2}:
return [{"drugtest_id": code, "AND_FLAG": None, "OR_FLAG": None}]
if code == 3:
return [
{"drugtest_id": 1, "AND_FLAG": 1, "OR_FLAG": None},
{"drugtest_id": 2, "AND_FLAG": 1, "OR_FLAG": None},
]
if code == 6:
return [{"drugtest_id": 96, "AND_FLAG": None, "OR_FLAG": None}]
if code == 7:
return [{"drugtest_id": 97, "AND_FLAG": None, "OR_FLAG": None}]
if code == 8:
return [{"drugtest_id": 98, "AND_FLAG": None, "OR_FLAG": None}]
if code == 9:
if 1993 <= year <= 2009:
return [
{"drugtest_id": 96, "AND_FLAG": None, "OR_FLAG": 1},
{"drugtest_id": 99, "AND_FLAG": None, "OR_FLAG": 1},
]
return [{"drugtest_id": 99, "AND_FLAG": None, "OR_FLAG": None}]
return []
if 2018 <= year <= 2022 and code in DRUGTEST_NODES:
return [{"drugtest_id": code, "AND_FLAG": None, "OR_FLAG": None}]
return []
def edge_row(
year: int,
person_id: str,
source_table: str,
source_slot: str,
source_drugtest_code: int | None,
mapped: dict[str, int | None],
) -> dict[str, Any]:
return {
"drug_event_id": f"{person_id}_DRUG_{source_slot}",
"drugtest_id": mapped["drugtest_id"],
"DATA_YEAR": year,
"source_table": source_table,
"source_slot": source_slot,
"source_drugtest_code": source_drugtest_code,
"AND_FLAG": mapped["AND_FLAG"],
"OR_FLAG": mapped["OR_FLAG"],
}
def iter_person_edges(base: Path, year: int) -> Iterator[dict[str, Any]]:
path = find_person_file(base, year)
if path is None:
print(f" WARNING {year}: no PERSON file found")
return
df = read_csv(path)
if year <= 1992:
pairs = [("1", "DRUGTEST", "DRUG_RES")]
else:
pairs = [
("1", "DRUGTST1", "DRUGRES1"),
("2", "DRUGTST2", "DRUGRES2"),
("3", "DRUGTST3", "DRUGRES3"),
]
required = {"ST_CASE", "VEH_NO", "PER_NO"}
missing = sorted(required - set(df.columns))
if missing:
print(f" WARNING {year}: {path.name} missing {missing}; skipping")
return
for rec in df.itertuples(index=False):
values = rec._asdict()
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
for slot, test_col, res_col in pairs:
if test_col not in df.columns or res_col not in df.columns:
continue
drugtest_code = int_code(values[test_col])
drugres_code = int_code(values[res_col])
if not has_drug_event(drugtest_code, drugres_code):
continue
for mapped in drugtest_edges_for(year, drugtest_code):
yield edge_row(year, person_id, "PERSON", slot, drugtest_code, mapped)
def iter_drugs_edges(base: Path, year: int) -> Iterator[dict[str, Any]]:
path = find_drugs_file(base, year)
if path is None:
print(f" WARNING {year}: no drugs.csv file found")
return
df = read_csv(path)
required = {"ST_CASE", "VEH_NO", "PER_NO", "DRUGSPEC", "DRUGRES"}
missing = sorted(required - set(df.columns))
if missing:
print(f" WARNING {year}: {path.name} missing {missing}; skipping")
return
person_sequence: Counter[str] = Counter()
for rec in df.itertuples(index=False):
values = rec._asdict()
drugtest_code = int_code(values["DRUGSPEC"])
drugres_code = int_code(values["DRUGRES"])
if not has_drug_event(drugtest_code, drugres_code):
continue
person_id = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
person_sequence[person_id] += 1
slot = str(person_sequence[person_id])
for mapped in drugtest_edges_for(year, drugtest_code):
yield edge_row(year, person_id, "DRUGS", slot, drugtest_code, mapped)
def iter_all_edges(base: Path) -> Iterator[dict[str, Any]]:
for year in range(1991, 2018):
yield from iter_person_edges(base, year)
for year in range(2018, 2023):
yield from iter_drugs_edges(base, year)
def dry_run(base: Path) -> None:
total = 0
counts_by_year: Counter[int] = Counter()
counts_by_drugtest: Counter[int] = Counter()
flags: Counter[str] = Counter()
start = time.time()
for row in iter_all_edges(base):
total += 1
counts_by_year[row["DATA_YEAR"]] += 1
counts_by_drugtest[row["drugtest_id"]] += 1
if row["AND_FLAG"] == 1:
flags["AND_FLAG"] += 1
if row["OR_FLAG"] == 1:
flags["OR_FLAG"] += 1
print(f"Dry run DRUG_TEST_OF edges prepared: {total}")
print(f"Counts by year: {dict(sorted(counts_by_year.items()))}")
print(f"Counts by DrugTest id: {dict(sorted(counts_by_drugtest.items()))}")
print(f"Flagged edge counts: {dict(sorted(flags.items()))}")
print(f"Dry-run elapsed seconds: {time.time() - start:.3f}")
def ensure_constraints_and_nodes(session: Any) -> None:
session.run(
"CREATE CONSTRAINT drugtest_id_unique IF NOT EXISTS "
"FOR (t:DrugTest) REQUIRE t.id IS UNIQUE"
).consume()
session.run(
"CREATE CONSTRAINT drugevent_id_unique IF NOT EXISTS "
"FOR (d:DrugEvent) REQUIRE d.drug_event_id IS UNIQUE"
).consume()
result = session.run(MERGE_DRUGTEST_NODES_QUERY, rows=DRUGTEST_NODE_ROWS).single()
print(f"Ensured DrugTest nodes: {result['matched'] if result else 0}")
def import_edges(session: Any, base: Path, batch_size: int) -> int:
total = 0
batch: list[dict[str, Any]] = []
start = time.time()
for row in iter_all_edges(base):
batch.append(row)
if len(batch) >= batch_size:
result = session.run(MERGE_EDGE_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported batch: total={total}")
batch = []
if batch:
result = session.run(MERGE_EDGE_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported final batch: total={total}")
print(f"Import loop elapsed seconds: {time.time() - start:.3f}")
return total
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started at {utc_now()}")
print(f"Base: {args.base}")
print(f"Database: {args.database}")
if args.dry_run:
dry_run(args.base)
print(f"Finished at {utc_now()}")
print(f"Total elapsed seconds: {time.time() - total_start:.3f}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints_and_nodes(session)
if args.ensure_constraints_only:
print("Constraints/nodes ensured; no DRUG_TEST_OF edges imported.")
return
imported = import_edges(session, args.base, args.batch_size)
print(f"Total DRUG_TEST_OF edges matched/created: {imported}")
finally:
driver.close()
total_elapsed = time.time() - total_start
print(f"Finished at {utc_now()}")
print(f"Total elapsed: {format_duration(total_elapsed)}")
print(f"Total elapsed seconds: {total_elapsed:.3f}")
if __name__ == "__main__":
main()
Other Factor to Vehicle Relationship (36.767s)
Average edge import time per year: 1.230s across 1975-1981.
Average edge import time per year: 0.715s across 1982-2009.
Average edge import time per year: 0.601s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS Factor-to-Vehicle relationships in Neo4j.
Sources:
* 1975-2009: vehicle-family VEH_CF1/VEH_CF2 columns from vehicle.csv and
vehnit.csv where present
* 2010-2019: factor.csv MFACTOR
* 2020-2022: factor.csv VEHICLECC
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:Factor)-[:FACTOR_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
FACTOR_NODES: dict[int, str] = {
1: "Tires",
2: "Brake System",
3: "Steering",
4: "Suspension",
5: "Power Train",
6: "Exhaust System",
7: "Head Lights",
8: "Signal Lights",
9: "Other Lights",
10: "Wipers",
11: "Wheels",
12: "Mirrors",
13: "Windows/Windshield",
14: "Body, Doors",
15: "Truck Coupling/Trailer Hitch/Safety Chains",
16: "Safety Systems",
17: "Vehicle Contributing Factors - No Details",
97: "Other",
98: "Not Reported",
99: "Unknown",
1001: "Horn",
1002: "Driver Seating and Control",
1003: "Air Bags",
1004: "Safety Belts",
}
FACTOR_NODE_ROWS = [{"id": factor_id, "name": name} for factor_id, name in FACTOR_NODES.items()]
VALID_FACTOR_IDS = set(FACTOR_NODES)
MERGE_FACTOR_NODES_QUERY = """
UNWIND $rows AS row
MERGE (f:Factor {id: row.id})
SET f.name = row.name
RETURN count(f) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (veh:Vehicle {vehicle_id: row.vehicle_id})
MATCH (factor:Factor {id: row.factor_id})
MERGE (factor)-[r:FACTOR_OF]->(veh)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "factor_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Factor-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Factor constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def legacy_factor_ids(year: int, code: int) -> list[int]:
if code == 0:
return []
if 1975 <= year <= 1981:
mapping = {
1: [1, 11],
2: [2],
3: [3],
4: [4],
5: [5],
6: [6],
7: [7],
8: [8],
9: [9],
10: [1001],
11: [12],
12: [10],
13: [1002],
14: [14],
15: [15],
99: [99],
}
return mapping.get(code, [])
if 1982 <= year <= 2009:
mapping = {
1: [1],
2: [2],
3: [3],
4: [4],
5: [5],
6: [6],
7: [7],
8: [8],
9: [9],
10: [1001],
11: [12],
12: [10],
13: [1002],
14: [14],
15: [15],
16: [11],
18: [97],
98: [98],
99: [99],
}
if 1995 <= year <= 2009 and code == 17:
return [1003]
if 2002 <= year <= 2009 and code == 19:
return [1004]
return mapping.get(code, [])
return []
def direct_factor_ids(code: int) -> list[int]:
if code == 0:
return []
if code in VALID_FACTOR_IDS:
return [code]
return []
def add_factor_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
factor_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, factor_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"factor_id": factor_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_source_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit))
return files
def find_factor_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["factor.csv", f"FACTOR_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "MFACTOR" in columns:
files.append((year, path, "MFACTOR"))
elif "VEHICLECC" in columns:
files.append((year, path, "VEHICLECC"))
return files
def iter_legacy_batches(
year: int,
source_table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "VEH_CF1", "VEH_CF2"}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in ("VEH_CF1", "VEH_CF2"):
code = int_code(record.get(source_column))
if code is None:
continue
for factor_id in legacy_factor_ids(year, code):
add_factor_edge(edge_rows, vehicle_id, year, factor_id, source_table, source_column, code)
yield list(edge_rows.values())
def iter_factor_batches(year: int, csv_path: Path, source_column: str, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
code = int_code(record.get(source_column))
if code is None:
continue
for factor_id in direct_factor_ids(code):
add_factor_edge(edge_rows, vehicle_id, year, factor_id, "FACTOR", source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path in find_legacy_source_files(base):
for rows in iter_legacy_batches(year, source_table, csv_path, chunk_size):
yield year, source_table, rows
for year, csv_path, source_column in find_factor_files(base):
for rows in iter_factor_batches(year, csv_path, source_column, chunk_size):
yield year, "FACTOR", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_factor: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_factor[int(row["factor_id"])] += 1
print(f"Expected FACTOR_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Factor id : {dict(sorted(rows_by_factor.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT factor_id IF NOT EXISTS FOR (f:Factor) REQUIRE f.id IS UNIQUE",
":Factor(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_FACTOR_NODES_QUERY, rows=FACTOR_NODE_ROWS).single()
print(f"Ensured Factor nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal FACTOR_OF rows sent: {rows_sent:,}")
print(f"Total FACTOR_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Maneuver to Vehicle Relationship (78.426s)
Average edge import time per year: 0.588s across 1982-2009.
Average edge import time per year: 4.766s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS Maneuver-to-Vehicle relationships in Neo4j.
Sources:
* 1982-2009: vehicle-family DR_CF1/DR_CF2/DR_CF3/DR_CF4 legacy columns
vehicle.csv, and vehnit.csv where present
* 2010-2019: maneuver.csv MDRMANAV
* 2020-2022: maneuver.csv MANEUVER
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:Maneuver)-[:MANEUVER_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
MANEUVER_NODES: dict[int, str] = {
0: "Driver Did Not Maneuver to Avoid",
1: "Object",
2: "Poor Road Conditions (Puddle, Ice, Pothole, etc.)",
3: "Live Animal",
4: "Contact Motor Vehicle (in This Crash)",
5: "Pedestrian, Pedalcyclist or Other Non-Motorist",
92: "Phantom/Non-Contact Motor Vehicle",
95: "No Driver Present/Unknown if Driver Present",
98: "Not Reported",
99: "Reported as Unknown",
}
MANEUVER_NODE_ROWS = [
{"id": maneuver_id, "name": name}
for maneuver_id, name in MANEUVER_NODES.items()
]
VALID_MANEUVER_IDS = set(MANEUVER_NODES)
MERGE_MANEUVER_NODES_QUERY = """
UNWIND $rows AS row
MERGE (m:Maneuver {id: row.id})
SET m.name = row.name
RETURN count(m) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
MATCH (m:Maneuver {id: row.maneuver_id})
MERGE (m)-[r:MANEUVER_OF]->(v)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "maneuver_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Maneuver-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Maneuver constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def map_legacy_driver_factor(code: int) -> int | None:
return {
81: 1, # Debris or objects in road.
82: 2, # Ruts, holes, bumps in road.
83: 3, # Live animals in road.
84: 4, # Vehicle in road.
85: 92, # Phantom vehicle.
86: 5, # Pedestrian, pedalcyclist, or other non-motorist in road.
}.get(code)
def add_maneuver_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
maneuver_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, maneuver_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"maneuver_id": maneuver_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_source_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year in range(1982, 2010):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit))
return files
def find_maneuver_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["maneuver.csv", f"MANEUVER_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "MANEUVER" in columns:
files.append((year, path, "MANEUVER"))
elif "MDRMANAV" in columns:
files.append((year, path, "MDRMANAV"))
return files
def iter_legacy_batches(
year: int,
source_table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
header = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
dr_columns = [column for column in ("DR_CF1", "DR_CF2", "DR_CF3", "DR_CF4") if column in columns]
wanted = {"ST_CASE", "VEH_NO", *dr_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in dr_columns:
code = int_code(record.get(source_column))
if code is None:
continue
maneuver_id = map_legacy_driver_factor(code)
if maneuver_id is None:
continue
add_maneuver_edge(edge_rows, vehicle_id, year, maneuver_id, source_table, source_column, code)
yield list(edge_rows.values())
def iter_maneuver_batches(
year: int,
csv_path: Path,
source_column: str,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
code = int_code(record.get(source_column))
if code is None or code not in VALID_MANEUVER_IDS:
continue
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
add_maneuver_edge(edge_rows, vehicle_id, year, code, "MANEUVER", source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path in find_legacy_source_files(base):
for rows in iter_legacy_batches(year, source_table, csv_path, chunk_size):
yield year, source_table, rows
for year, csv_path, source_column in find_maneuver_files(base):
for rows in iter_maneuver_batches(year, csv_path, source_column, chunk_size):
yield year, "MANEUVER", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_maneuver: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_maneuver[int(row["maneuver_id"])] += 1
print(f"Expected MANEUVER_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Maneuver id : {dict(sorted(rows_by_maneuver.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT maneuver_id IF NOT EXISTS FOR (m:Maneuver) REQUIRE m.id IS UNIQUE",
":Maneuver(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if rows:
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_MANEUVER_NODES_QUERY, rows=MANEUVER_NODE_ROWS).consume()
print("Ensured Maneuver category nodes")
if args.ensure_constraints_only:
print(f"Maneuver constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Maneuver edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:MANEUVER_OF]->() RETURN count(r) AS count").single()["count"]
print(f"Maneuver edge load finished UTC : {utc_now()}")
print(f"Maneuver edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"MANEUVER_OF relationship count : {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Non-Motorist Crash Factor to Person Relationship (51.928s)
Average edge import time per year: 1.200s across 1975-2009.
Average edge import time per year: 0.722s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS NMCrash-to-Person relationships in Neo4j.
Sources:
* 1975-2009: person.csv P_CF1/P_CF2/P_CF3 recoded according to nmcrash.xlsx
* 2010-2019: nmcrash.csv MTM_CRSH
* 2020-2022: nmcrash.csv NMCC
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. Raw FARS
identifiers are not globally unique across years.
Relationship:
(:NMCrash)-[:NMCRASH_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
NMCRASH_NODES: dict[int, str] = {
1: "Dart-out - Visual Obstruction Noted",
2: "Failure to Yield Right-Of-Way",
3: "Failure to Obey Traffic Signs, Signals or Officer",
4: "In Roadway Improperly (Standing, Lying, Working, Playing)",
5: "Entering/Exiting Parked or Stopped Vehicle",
6: "Inattentive (Talking, Eating, etc.)",
7: "Improper Turn/Merge",
8: "Improper Passing",
9: "Wrong-Way Riding or Walking",
10: "Riding on Wrong Side of Road",
11: "Dash - Run, No Visual Obstruction Noted",
12: "Improper Crossing of Roadway or Intersection (Jaywalking)",
13: "Failing to Have Lights on When Required",
14: "Operating Without Required Equipment",
15: "Improper or Erratic Lane Changing",
16: "Failure to Keep in Proper Lane or Running off Road",
17: "Making Improper Entry to or Exit From Trafficway",
18: "Operating in Other Erratic, Reckless, Careless or Negligent Manner",
19: "Not Visible (Dark Clothing, No Lighting, etc.)",
20: "Passing With Insufficient Distance or Inadequate Visibility or Failing to Yield to Overtaking Vehicle",
21: "Other",
92: "Contributing Circumstance -No Details",
98: "Not Reported",
99: "Reported as Unknown",
}
NMCRASH_NODE_ROWS = [{"id": code, "name": name} for code, name in NMCRASH_NODES.items()]
VALID_NMCRASH_IDS = set(NMCRASH_NODES)
MERGE_NMCRASH_NODES_QUERY = """
UNWIND $rows AS row
MERGE (n:NMCrash {id: row.id})
SET n.name = row.name
RETURN count(n) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MATCH (n:NMCrash {id: row.nmcrash_id})
MERGE (n)-[r:NMCRASH_OF]->(p)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "nmcrash_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS NMCrash-to-Person relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure NMCrash constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def legacy_nmcrash_ids(year: int, code: int) -> list[int]:
if code == 0:
return []
if 1975 <= year <= 1981:
mapping = {
2: [19],
3: [1],
4: [12],
23: [13],
27: [15],
28: [16],
30: [17],
35: [20],
48: [7],
49: [9],
50: [10],
99: [99],
}
return mapping.get(code, [])
if 1982 <= year <= 2009:
mapping = {
1: [19],
3: [12],
10: [6],
23: [13],
27: [15],
28: [16],
30: [17],
35: [20],
38: [2],
48: [7],
49: [9],
99: [99],
}
if year >= 1995 and code == 2:
return [1]
if year >= 1995 and code == 50:
return [10]
if 1982 <= year <= 1994 and code == 84:
return [12]
return mapping.get(code, [])
return []
def direct_nmcrash_ids(code: int) -> list[int]:
if code == 0:
return []
if code in VALID_NMCRASH_IDS:
return [code]
return []
def add_nmcrash_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
person_id: str,
year: int,
nmcrash_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (person_id, nmcrash_id)
row = edge_rows.setdefault(
key,
{
"person_id": person_id,
"nmcrash_id": nmcrash_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_person_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
path = find_file(national, ["person.csv", f"PERSON_{year}.CSV"])
if path is not None:
files.append((year, path))
return files
def find_nmcrash_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["nmcrash.csv", f"NMCRASH_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "NMCC" in columns:
files.append((year, path, "NMCC"))
elif "MTM_CRSH" in columns:
files.append((year, path, "MTM_CRSH"))
return files
def iter_legacy_batches(year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
source_columns = ("P_CF1", "P_CF2", "P_CF3")
wanted = {"ST_CASE", "VEH_NO", "PER_NO", *source_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
for source_column in source_columns:
code = int_code(record.get(source_column))
if code is None:
continue
for nmcrash_id in legacy_nmcrash_ids(year, code):
add_nmcrash_edge(edge_rows, person_id, year, nmcrash_id, "PERSON", source_column, code)
yield list(edge_rows.values())
def iter_nmcrash_batches(year: int, csv_path: Path, source_column: str, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "PER_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
code = int_code(record.get(source_column))
if code is None:
continue
for nmcrash_id in direct_nmcrash_ids(code):
add_nmcrash_edge(edge_rows, person_id, year, nmcrash_id, "NMCRASH", source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path in find_legacy_person_files(base):
for rows in iter_legacy_batches(year, csv_path, chunk_size):
yield year, "PERSON", rows
for year, csv_path, source_column in find_nmcrash_files(base):
for rows in iter_nmcrash_batches(year, csv_path, source_column, chunk_size):
yield year, "NMCRASH", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_nmcrash: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_nmcrash[int(row["nmcrash_id"])] += 1
print(f"Expected NMCRASH_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by NMCrash id : {dict(sorted(rows_by_nmcrash.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT nmcrash_id IF NOT EXISTS FOR (n:NMCrash) REQUIRE n.id IS UNIQUE",
":NMCrash(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT person_person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_NMCRASH_NODES_QUERY, rows=NMCRASH_NODE_ROWS).single()
print(f"Ensured NMCrash nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal NMCRASH_OF rows sent: {rows_sent:,}")
print(f"Total NMCRASH_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Non-Motorist Distraction to Person Relationship (4.129s)
Average edge import time per year: 0.936s across 2019-2022.
python
#!/usr/bin/env python3
"""
Create FARS NMDistract-to-Person relationships in Neo4j.
Sources:
* 2019: nmdistract.csv MNMDSTRD
* 2020-2022: nmdistract.csv NMDISTRACT
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. Raw FARS
identifiers are not globally unique across years.
Relationship:
(:NMDistract)-[:NMDISTRACT_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
NMDISTRACT_NODES: dict[int, str] = {
2: "By Other Non-Motorist(s)",
3: "By a Driver or Occupant of a Motor Vehicle",
5: "While Talking or Listening to Mobile Phone",
6: "While Manipulating Mobile Phone",
7: "Adjusting or Listening to Portable Audio Device (Other Than on a Mobile Phone)",
8: "Adjusting, Talking to, or Manipulating Other Portable Electronic Device",
12: "Distracted by Animal, Other Object, Event, or Activity",
13: "Eating or Drinking",
14: "Smoking Related",
15: "Other Mobile Phone Related",
17: "Distraction/Inattention",
18: "Distraction/Careless",
19: "Careless/Inattentive",
92: "Distraction (Distracted), Details Unknown",
93: "Inattention (Inattentive), Details Unknown",
96: "Not Reported",
97: "Lost in Thought/Daydreaming",
98: "Other Distraction",
99: "Reported as Unknown if Distracted",
}
NMDISTRACT_NODE_ROWS = [{"id": code, "name": name} for code, name in NMDISTRACT_NODES.items()]
VALID_NMDISTRACT_IDS = set(NMDISTRACT_NODES)
MERGE_NMDISTRACT_NODES_QUERY = """
UNWIND $rows AS row
MERGE (n:NMDistract {id: row.id})
SET n.name = row.name
RETURN count(n) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MATCH (n:NMDistract {id: row.nmdistract_id})
MERGE (n)-[r:NMDISTRACT_OF]->(p)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "nmdistract_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS NMDistract-to-Person relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure NMDistract constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def direct_nmdistract_ids(code: int) -> list[int]:
if code == 0:
return []
if code in VALID_NMDISTRACT_IDS:
return [code]
return []
def add_nmdistract_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
person_id: str,
year: int,
nmdistract_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (person_id, nmdistract_id)
row = edge_rows.setdefault(
key,
{
"person_id": person_id,
"nmdistract_id": nmdistract_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_nmdistract_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2019, 2023):
national = base / str(year) / "National"
path = find_file(national, ["nmdistract.csv", f"NMDISTRACT_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "NMDISTRACT" in columns:
files.append((year, path, "NMDISTRACT"))
elif "MNMDSTRD" in columns:
files.append((year, path, "MNMDSTRD"))
return files
def iter_nmdistract_batches(year: int, csv_path: Path, source_column: str, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "PER_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
code = int_code(record.get(source_column))
if code is None:
continue
for nmdistract_id in direct_nmdistract_ids(code):
add_nmdistract_edge(edge_rows, person_id, year, nmdistract_id, "NMDISTRACT", source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path, source_column in find_nmdistract_files(base):
for rows in iter_nmdistract_batches(year, csv_path, source_column, chunk_size):
yield year, "NMDISTRACT", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_nmdistract: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_nmdistract[int(row["nmdistract_id"])] += 1
print(f"Expected NMDISTRACT_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by NMDistract id : {dict(sorted(rows_by_nmdistract.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT nmdistract_id IF NOT EXISTS FOR (n:NMDistract) REQUIRE n.id IS UNIQUE",
":NMDistract(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT person_person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_NMDISTRACT_NODES_QUERY, rows=NMDISTRACT_NODE_ROWS).single()
print(f"Ensured NMDistract nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal NMDISTRACT_OF rows sent: {rows_sent:,}")
print(f"Total NMDISTRACT_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Non-Motorist Impairment to Person Relationship (35.800s)
Average edge import time per year: 0.865s across 1975-1981.
Average edge import time per year: 0.793s across 1982-2009.
Average edge import time per year: 0.783s across 2019-2022.
python
#!/usr/bin/env python3
"""
Create FARS NMImpair-to-Person relationships in Neo4j.
Sources:
* 1975-1981: person.csv P_CF1/P_CF2/P_CF3 code 1 recoded to NMImpair 10
* 1982-2009: person.csv P_CF1/P_CF2/P_CF3 selected legacy impairment codes
* 2010-2022: nmimpair.csv NMIMPAIR
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. Raw FARS
identifiers are not globally unique across years.
Relationship:
(:NMImpair)-[:NMIMPAIR_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
NMIMPAIR_NODES: dict[int, str] = {
1: "Ill, Blackout",
2: "Asleep or Fatigued",
3: "Walking With a Cane or Crutches",
4: "Paraplegic or in a Wheelchair",
5: "Impaired Due to Previous Injury",
6: "Deaf/Hard of Hearing",
7: "Blind/Low Vision",
8: "Emotional (Depressed, Angry, Disturbed, etc.)",
9: "Under the Influence of Alcohol, Drugs, or Medication",
10: "Physical Impairment - No Details",
96: "Other Physical Impairment",
98: "Not Reported",
99: "Reported as Unknown if Impaired",
}
NMIMPAIR_NODE_ROWS = [{"id": code, "name": name} for code, name in NMIMPAIR_NODES.items()]
VALID_NMIMPAIR_IDS = set(NMIMPAIR_NODES)
MERGE_NMIMPAIR_NODES_QUERY = """
UNWIND $rows AS row
MERGE (n:NMImpair {id: row.id})
SET n.name = row.name
RETURN count(n) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MATCH (n:NMImpair {id: row.nmimpair_id})
MERGE (n)-[r:NMIMPAIR_OF]->(p)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "nmimpair_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS NMImpair-to-Person relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure NMImpair constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def legacy_nmimpair_ids(year: int, code: int) -> list[int]:
if code == 0:
return []
if 1975 <= year <= 1981:
return [10] if code == 1 else []
if 1982 <= year <= 2009:
mapping = {
7: [8],
11: [3],
12: [4],
14: [5],
16: [7],
17: [96],
}
if year >= 1995 and code == 6:
return [1]
if year <= 1994 and code == 13:
return [4]
return mapping.get(code, [])
return []
def direct_nmimpair_ids(code: int) -> list[int]:
if code == 0:
return []
if code in VALID_NMIMPAIR_IDS:
return [code]
return []
def add_nmimpair_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
person_id: str,
year: int,
nmimpair_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (person_id, nmimpair_id)
row = edge_rows.setdefault(
key,
{
"person_id": person_id,
"nmimpair_id": nmimpair_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_person_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
path = find_file(national, ["person.csv", f"PERSON_{year}.CSV"])
if path is not None:
files.append((year, path))
return files
def find_nmimpair_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["nmimpair.csv", f"NMIMPAIR_{year}.CSV"])
if path is not None:
files.append((year, path))
return files
def iter_legacy_batches(year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
source_columns = ("P_CF1", "P_CF2", "P_CF3")
wanted = {"ST_CASE", "VEH_NO", "PER_NO", *source_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
for source_column in source_columns:
code = int_code(record.get(source_column))
if code is None:
continue
for nmimpair_id in legacy_nmimpair_ids(year, code):
add_nmimpair_edge(edge_rows, person_id, year, nmimpair_id, "PERSON", source_column, code)
yield list(edge_rows.values())
def iter_nmimpair_batches(year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "PER_NO", "NMIMPAIR"}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
code = int_code(record.get("NMIMPAIR"))
if code is None:
continue
for nmimpair_id in direct_nmimpair_ids(code):
add_nmimpair_edge(edge_rows, person_id, year, nmimpair_id, "NMIMPAIR", "NMIMPAIR", code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path in find_legacy_person_files(base):
for rows in iter_legacy_batches(year, csv_path, chunk_size):
yield year, "PERSON", rows
for year, csv_path in find_nmimpair_files(base):
for rows in iter_nmimpair_batches(year, csv_path, chunk_size):
yield year, "NMIMPAIR", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_nmimpair: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_nmimpair[int(row["nmimpair_id"])] += 1
print(f"Expected NMIMPAIR_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by NMImpair id : {dict(sorted(rows_by_nmimpair.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT nmimpair_id IF NOT EXISTS FOR (n:NMImpair) REQUIRE n.id IS UNIQUE",
":NMImpair(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT person_person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_NMIMPAIR_NODES_QUERY, rows=NMIMPAIR_NODE_ROWS).single()
print(f"Ensured NMImpair nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal NMIMPAIR_OF rows sent: {rows_sent:,}")
print(f"Total NMIMPAIR_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Non-Motorist Prior Action to Person Relationship (23.520s)
Average edge import time per year: 0.694s across 1982-2009.
Average edge import time per year: 1.092s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS NMPrior-to-Person relationships in Neo4j.
Sources:
* 1982-1994: person.csv P_CF1/P_CF2/P_CF3 code 85 recoded to NMPrior 4
* 2010-2019: nmprior.csv MPR_ACT
* 2020-2022: nmprior.csv NMACTION
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. Raw FARS
identifiers are not globally unique across years.
Relationship:
(:NMPrior)-[:NMPRIOR_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
NMPRIOR_NODES: dict[int, str] = {
1: "Going to or From School [Pre-K-12] (Since 2023)",
2: "Waiting to Cross Roadway",
3: "Crossing Roadway",
4: "Jogging/Running",
5: "Movement Along Roadway With Traffic (in or Adjacent to Travel Lane)",
6: "Movement Along Roadway Against Traffic (in or Adjacent to Travel Lane)",
7: "Movement on Sidewalk",
8: "In Roadway-Other [Working, Playing, etc.]",
9: "Adjacent to Roadway (e.g., Shoulder, Median)",
10: "Working in Trafficway [Incident Response]",
11: "Entering/Exiting a Vehicle",
12: "Disabled Vehicle Related (Working on, Pushing, Leaving/Approaching)",
14: "Other",
15: "None",
16: "Movement Along Roadway - Direction Unknown (Since 2012)",
98: "Not Reported",
99: "Unknown",
}
NMPRIOR_NODE_ROWS = [{"id": code, "name": name} for code, name in NMPRIOR_NODES.items()]
VALID_NMPRIOR_IDS = set(NMPRIOR_NODES)
MERGE_NMPRIOR_NODES_QUERY = """
UNWIND $rows AS row
MERGE (n:NMPrior {id: row.id})
SET n.name = row.name
RETURN count(n) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MATCH (n:NMPrior {id: row.nmprior_id})
MERGE (n)-[r:NMPRIOR_OF]->(p)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "nmprior_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS NMPrior-to-Person relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure NMPrior constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def legacy_nmprior_ids(year: int, code: int) -> list[int]:
if 1982 <= year <= 1994 and code == 85:
return [4]
return []
def direct_nmprior_ids(code: int) -> list[int]:
if code == 0:
return []
if code in VALID_NMPRIOR_IDS:
return [code]
return []
def add_nmprior_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
person_id: str,
year: int,
nmprior_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (person_id, nmprior_id)
row = edge_rows.setdefault(
key,
{
"person_id": person_id,
"nmprior_id": nmprior_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_person_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year in range(1982, 1995):
national = base / str(year) / "National"
path = find_file(national, ["person.csv", f"PERSON_{year}.CSV"])
if path is not None:
files.append((year, path))
return files
def find_nmprior_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["nmprior.csv", f"NMPRIOR_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "NMACTION" in columns:
files.append((year, path, "NMACTION"))
elif "MPR_ACT" in columns:
files.append((year, path, "MPR_ACT"))
return files
def iter_legacy_batches(year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
source_columns = ("P_CF1", "P_CF2", "P_CF3")
wanted = {"ST_CASE", "VEH_NO", "PER_NO", *source_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
for source_column in source_columns:
code = int_code(record.get(source_column))
if code is None:
continue
for nmprior_id in legacy_nmprior_ids(year, code):
add_nmprior_edge(edge_rows, person_id, year, nmprior_id, "PERSON", source_column, code)
yield list(edge_rows.values())
def iter_nmprior_batches(year: int, csv_path: Path, source_column: str, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "PER_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
code = int_code(record.get(source_column))
if code is None:
continue
for nmprior_id in direct_nmprior_ids(code):
add_nmprior_edge(edge_rows, person_id, year, nmprior_id, "NMPRIOR", source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path in find_legacy_person_files(base):
for rows in iter_legacy_batches(year, csv_path, chunk_size):
yield year, "PERSON", rows
for year, csv_path, source_column in find_nmprior_files(base):
for rows in iter_nmprior_batches(year, csv_path, source_column, chunk_size):
yield year, "NMPRIOR", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_nmprior: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_nmprior[int(row["nmprior_id"])] += 1
print(f"Expected NMPRIOR_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by NMPrior id : {dict(sorted(rows_by_nmprior.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT nmprior_id IF NOT EXISTS FOR (n:NMPrior) REQUIRE n.id IS UNIQUE",
":NMPrior(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT person_person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_NMPRIOR_NODES_QUERY, rows=NMPRIOR_NODE_ROWS).single()
print(f"Ensured NMPrior nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal NMPRIOR_OF rows sent: {rows_sent:,}")
print(f"Total NMPRIOR_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Person Related Factor to Person Relationship (11.770s)
This import is performed as one total batch rather than by year, so no yearly average time is available.
python
#!/usr/bin/env python3
"""
Create FARS PersonRF-to-Person relationships in Neo4j.
Sources:
* 1975-2009: person.csv P_CF1/P_CF2/P_CF3
* 2010-2019: person.csv P_SF1/P_SF2/P_SF3
* 2020-2022: personrf.csv PERSONRF
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. Raw FARS
identifiers are not globally unique across years.
Relationship:
(:PersonRF)-[:PERSON_RF_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
PERSONRF_NODES: dict[int, str] = {
4: "Walking/Riding With or Against Traffic, Playing, Working, Sitting, Lying, Standing, etc., in Roadway",
5: "Interfering With Driver*",
8: "Person with an Intellectual, Cognitive, or Developmental Disability",
9: "Construction/Maintenance/Utility Worker",
10: "Alcohol and/or Drug Test Refused (Since 2017)",
13: "Motorized Wheelchair Rider**",
18: "Mother of Dead Fetus/Mother of Infant Born Post Crash",
19: "Pedestrian",
20: "Leaving Vehicle Unattended in Roadway",
21: "Overloading or Improper Loading of Vehicle With Passengers or Cargo",
22: "Towing or Pushing Vehicle Improperly",
24: "Operating Without Required Equipment",
25: "Creating Unlawful Noise or Using Equipment Prohibited by Law",
26: "Following Improperly",
29: "Illegal Driving on Road Shoulder, in Ditch, on Sidewalk, on Median",
31: "Default Code Used for Vehicle Numbering**",
32: "Opening Vehicle Closure Into Moving Traffic or While Vehicle is in Motion (Since 2001)*",
33: "Passing Where Prohibited by Posted Signs, Pavement Markings, Hill, or Curve, or School Bus Displaying Warning Not to Pass",
34: "Passing on Wrong Side",
36: "Operating the Vehicle in Other Erratic, Reckless, Careless or Negligent Manner",
37: "Traveling on Prohibited Trafficway",
39: "Failure to Obey Actual Traffic Signs, Traffic Control Devices or Traffic Officers; Failure to Obey Safety Zone Traffic Laws",
40: "Passing Through or Around Barrier Positioned to Prohibit or Channel Traffic",
41: "Failure to Observe Warnings or Instructions on Vehicles Displaying Them",
42: "Failure to Signal Intentions",
43: "Giving Wrong Signal",
44: "Driving Too Fast for Conditions or in Excess of Posted Speed Limit",
45: "Driving Less Than Posted Maximum",
46: "Operating at Erratic or Suddenly Changing Speeds",
47: "Making Right Turn From Left Turn Lane or Making Left Turn From Right Turn Lane",
51: "Operator Inexperience",
52: "Unfamiliar With Roadway",
53: "Non-Motorist Previously Used a Motor Vehicle for Motion**",
54: "Non-Motorist Attempting to Use a Motor Vehicle for Motion**",
55: "Non-Motorist Attempting to Use or Previously Used a Motor Vehicle for Motion, Details Not Reported**",
56: "Non-Driver Flees Scene (Since 2005)",
57: "Improper Tire Pressure (Since 1995)",
58: "Locked Wheel (1995-2014)",
59: "Overcorrecting (Since 1995)*",
60: "Rain, Snow, Fog, Smoke, Sand, Dust",
61: "Reflected Glare, Bright Sunlight, Headlights",
62: "Curve, Hill, or Other Design Features (Including Traffic Signs, Embankment)",
63: "Building, Billboard, Other Structures",
64: "Trees, Crops, Vegetation",
65: "Motor Vehicle (Including Load)",
66: "Parked Vehicle",
67: "Splash or Spray or Passing Vehicle",
68: "Inadequate Lighting System",
69: "Obstructing Angles on Vehicle",
70: "Mirrors",
71: "Mirrors-Other",
72: "Other Visual Obstruction",
73: "Severe Crosswind",
74: "Wind From Passing Truck",
75: "Slippery or Loose Surface",
76: "Tire Blow-Out or Flat",
77: "Debris or Objects in Road",
78: "Ruts, Holes, Bumps in Road",
79: "Live Animals in Road",
80: "Vehicle in Road",
81: "Phantom Vehicle",
82: "Pedestrian, Pedalcyclist, or Other Non-Motorist",
83: "Ice, Snow, Slush, Water, Sand, Dirt, Oil, Wet Leaves on Road (Since 1995)",
86: "Emergency Services Personnel (Since 2007)",
87: "Police or Law Enforcement Officer",
88: "Seat Back Not in Normal Upright Position, Seat Back Reclined*",
89: "Parked Motor Vehicle With Equipment Extending Into the Travel Lane",
90: "Non-Motorist Pushing a Vehicle**",
91: "Portable Electronic Devices",
92: "Person in Ambulance Treatment Compartment*",
93: "Non-Motorist Wearing Motorcycle Helmet**",
94: "Emergency Medical Services Personnel",
95: "Fire Personnel",
96: "Tow Operator",
97: "Transportation (Maintenance Workers, Safety Service Patrol Operators, etc.)",
100: "Using a Shared Micromobility Device**",
101: "Obstructed Sidewalk (for this Person)**",
999: "Reported as Unknown",
1001: "Stopping in Roadway (Vehicle Not Abandoned)",
1002: "Underriding a Parked Truck (1982-1996)",
1003: "Getting off/out of or on/into Moving Transport Vehicle",
}
PERSONRF_NODE_ROWS = [{"id": code, "name": name} for code, name in PERSONRF_NODES.items()]
VALID_PERSONRF_IDS = set(PERSONRF_NODES)
MERGE_PERSONRF_NODES_QUERY = """
UNWIND $rows AS row
MERGE (n:PersonRF {id: row.id})
SET n.name = row.name
RETURN count(n) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MATCH (n:PersonRF {id: row.personrf_id})
MERGE (n)-[r:PERSON_RF_OF]->(p)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "personrf_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS PersonRF-to-Person relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure PersonRF constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def personrf_ids_for(year: int, code: int) -> list[int]:
if code == 0:
return []
if 1975 <= year <= 1981:
mapping = {
5: [4],
6: [5],
20: [20],
21: [21],
22: [22],
24: [24],
25: [25],
26: [26],
29: [29],
33: [33],
34: [34],
36: [36],
40: [40],
41: [41],
42: [42],
43: [43],
44: [44],
45: [45],
46: [46],
47: [47],
51: [51],
52: [52],
99: [999],
}
return mapping.get(code, [])
if 1982 <= year <= 2009:
if code == 53:
return [1001]
if code == 54:
return [1002]
if code == 55:
return [1003]
if code == 99:
return [999]
valid = {
4,
5,
18,
19,
21,
24,
26,
29,
33,
34,
36,
39,
40,
41,
42,
44,
45,
47,
51,
52,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
82,
87,
88,
90,
91,
}
if 1982 <= year <= 1994:
valid.add(20)
if 1982 <= year <= 2003:
valid.add(22)
if 1982 <= year <= 2002:
valid.add(25)
if 1982 <= year <= 1996:
valid.update({43, 46})
if year >= 1995:
valid.update({8, 9, 37, 57, 58, 59, 83})
if year >= 2001:
valid.add(32)
if year >= 2005:
valid.add(56)
if year >= 2007:
valid.add(86)
if code in valid and code in VALID_PERSONRF_IDS:
return [code]
return []
if 2010 <= year <= 2019:
if code == 99:
return [999]
if code == 10 and year < 2017:
return []
if code == 31 and year < 2018:
return []
if code in {53, 54, 55} and year < 2019:
return []
if code == 58 and year > 2014:
return []
if code == 86 and year > 2018:
return []
if code == 93 and year < 2016:
return []
if code in {94, 95, 96, 97} and year < 2019:
return []
valid = {
5,
8,
9,
10,
13,
18,
21,
26,
29,
31,
32,
33,
37,
40,
41,
42,
44,
45,
47,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
72,
73,
74,
75,
76,
77,
78,
80,
81,
82,
83,
86,
87,
88,
89,
90,
91,
92,
93,
94,
95,
96,
97,
}
if code in valid and code in VALID_PERSONRF_IDS:
return [code]
return []
if 2020 <= year <= 2022:
if code in VALID_PERSONRF_IDS:
return [code]
return []
return []
def add_personrf_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
person_id: str,
year: int,
personrf_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (person_id, personrf_id)
row = edge_rows.setdefault(
key,
{
"person_id": person_id,
"personrf_id": personrf_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_person_files(base: Path) -> list[tuple[int, Path, str, list[str]]]:
files: list[tuple[int, Path, str, list[str]]] = []
for year in range(1975, 2023):
national = base / str(year) / "National"
if year <= 2019:
path = find_file(national, [f"PERSON_{year}.CSV", "person.csv", "PERSON.CSV", "Person.csv"])
if path is not None:
source_columns = ["P_CF1", "P_CF2", "P_CF3"] if year <= 2009 else ["P_SF1", "P_SF2", "P_SF3"]
files.append((year, path, "PERSON", source_columns))
else:
path = find_file(
national,
[
f"PERSONRF_{year}.CSV",
"personrf.csv",
"PERSONRF.CSV",
"PersonRF.csv",
"Personrf.csv",
],
)
if path is not None:
files.append((year, path, "PERSONRF", ["PERSONRF"]))
return files
def read_csv(path: Path) -> pd.DataFrame:
for encoding in ("utf-8", "latin-1"):
try:
df = pd.read_csv(path, encoding=encoding, low_memory=False)
df.columns = normalize_columns(df.columns)
return df
except UnicodeDecodeError:
continue
raise UnicodeDecodeError("utf-8", b"", 0, 1, f"Could not decode {path}")
def collect_edges(base: Path) -> tuple[list[dict[str, Any]], Counter[int], Counter[int]]:
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
raw_counts: Counter[int] = Counter()
mapped_counts: Counter[int] = Counter()
files = find_person_files(base)
for year, path, source_table, source_columns in files:
start = time.time()
df = read_csv(path)
required = {"ST_CASE", "VEH_NO", "PER_NO"}
missing = sorted(required - set(df.columns))
if missing:
print(f" WARNING {year}: {path.name} missing required columns {missing}; skipping")
continue
available_source_columns = [column for column in source_columns if column in df.columns]
if not available_source_columns:
print(f" WARNING {year}: {path.name} has none of {source_columns}; skipping")
continue
year_edges_before = len(edge_rows)
for rec in df[["ST_CASE", "VEH_NO", "PER_NO", *available_source_columns]].itertuples(index=False):
values = rec._asdict()
pid = person_id_for(year, values["ST_CASE"], values["VEH_NO"], values["PER_NO"])
if not pid or pid.endswith("__"):
continue
for source_column in available_source_columns:
code = int_code(values[source_column])
if code is None:
continue
raw_counts[code] += 1
for personrf_id in personrf_ids_for(year, code):
add_personrf_edge(edge_rows, pid, year, personrf_id, source_table, source_column, code)
mapped_counts[personrf_id] += 1
print(
f" Prepared {year}: {len(edge_rows) - year_edges_before} unique edges "
f"from {path.name} in {format_duration(time.time() - start)}"
)
return list(edge_rows.values()), raw_counts, mapped_counts
def ensure_constraints_and_nodes(session: Any) -> None:
session.run(
"CREATE CONSTRAINT personrf_id_unique IF NOT EXISTS "
"FOR (n:PersonRF) REQUIRE n.id IS UNIQUE"
).consume()
session.run(
"CREATE CONSTRAINT person_person_id_unique IF NOT EXISTS "
"FOR (p:Person) REQUIRE p.person_id IS UNIQUE"
).consume()
result = session.run(MERGE_PERSONRF_NODES_QUERY, rows=PERSONRF_NODE_ROWS).single()
print(f"Ensured PersonRF nodes: {result['matched'] if result else 0}")
def import_edges(session: Any, rows: list[dict[str, Any]], batch_size: int) -> int:
total = 0
for i in range(0, len(rows), batch_size):
batch = rows[i : i + batch_size]
result = session.run(MERGE_EDGE_QUERY, rows=batch).single()
total += result["matched"] if result else 0
print(f" Imported batch {i // batch_size + 1}: {len(batch)} rows")
return total
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started at {utc_now()}")
print(f"Base: {args.base}")
print(f"Database: {args.database}")
if args.dry_run:
rows, raw_counts, mapped_counts = collect_edges(args.base)
print(f"\nDry run only. Unique PERSON_RF_OF edges prepared: {len(rows)}")
print(f"Mapped PersonRF ids: {dict(sorted(mapped_counts.items()))}")
unmapped = {code: count for code, count in sorted(raw_counts.items()) if not any(personrf_ids_for(y, code) for y in range(1975, 2023))}
if unmapped:
print(f"Raw codes with no PersonRF mapping in any year: {unmapped}")
print(f"Finished at {utc_now()}")
print(f"Total elapsed seconds: {time.time() - total_start:.3f}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints_and_nodes(session)
if args.ensure_constraints_only:
print("Constraints/nodes ensured; no relationships imported.")
return
rows, raw_counts, mapped_counts = collect_edges(args.base)
print(f"\nPrepared unique PERSON_RF_OF edges: {len(rows)}")
print(f"Mapped PersonRF ids: {dict(sorted(mapped_counts.items()))}")
start = time.time()
imported = import_edges(session, rows, args.batch_size)
print(f"\nTotal PERSON_RF_OF edges matched/created: {imported}")
print(f"Import elapsed seconds: {time.time() - start:.3f}")
finally:
driver.close()
total_elapsed = time.time() - total_start
print(f"Finished at {utc_now()}")
print(f"Total elapsed: {format_duration(total_elapsed)}")
print(f"Total elapsed seconds: {total_elapsed:.3f}")
if __name__ == "__main__":
main()
Race to Person Relationship (97.395s)
Average edge import time per year: 4.058s across 1999-2022.
python
#!/usr/bin/env python3
"""
Create FARS Race-to-Person relationships in Neo4j.
Sources:
* 1999-2018: person.csv RACE
* 2019-2022: race.csv RACE
The graph join uses person_id = <year>_<ST_CASE>_<VEH_NO>_<PER_NO>. Raw
ST_CASE/VEH_NO/PER_NO values are not globally unique across years.
Relationship:
(:Race)-[:RACE_OF]->(:Person)
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
RACE_NODES: dict[int, str] = {
0: "Not a Fatality (Not Applicable)",
1: "White",
2: "Black or African American",
3: "North American Indian or Alaska Native",
4: "Chinese",
5: "Japanese",
6: "Native Hawaiian",
7: "Filipino",
18: "Asian Indian",
19: "Other Indian",
28: "Korean",
38: "Samoan",
48: "Vietnamese",
58: "Guamanian or Chamorro",
68: "Other Asian or Pacific Islander",
78: "Asian or Pacific Islander, No Specific (Individual) Race",
96: "Redacted",
97: "Multiple Races, Unspecified",
98: "All Other Races",
99: "Unknown",
}
RACE_NODE_ROWS = [
{"id": race_id, "name": name}
for race_id, name in RACE_NODES.items()
]
# Code 0 is a valid source value but means race is not applicable for nonfatal
# persons, so it is kept as a category node but not linked as a race edge.
VALID_RACE_EDGE_IDS = set(RACE_NODES) - {0}
MERGE_RACE_NODES_QUERY = """
UNWIND $rows AS row
MERGE (r:Race {id: row.id})
SET r.name = row.name
RETURN count(r) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (p:Person {person_id: row.person_id})
MATCH (r:Race {id: row.race_id})
MERGE (r)-[rel:RACE_OF]->(p)
SET rel.DATA_YEAR = row.DATA_YEAR,
rel.source_table = row.source_table,
rel.source_column = row.source_column,
rel.source_code = row.source_code,
rel.multrace = row.multrace,
rel.created_from = "race_edges"
RETURN count(rel) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Race-to-Person relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Race constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def person_id_for(year: int, st_case: Any, veh_no: Any, per_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}_{identifier_part(per_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def find_source_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year in range(1999, 2019):
national = base / str(year) / "National"
path = find_file(national, ["person.csv", f"PERSON_{year}.CSV"])
if path is not None:
files.append((year, "PERSON", path))
for year in range(2019, 2023):
national = base / str(year) / "National"
path = find_file(national, ["race.csv", f"RACE_{year}.CSV"])
if path is not None:
files.append((year, "RACE", path))
return files
def add_race_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
person_id: str,
year: int,
race_id: int,
source_table: str,
source_code: int,
multrace: int | None,
) -> None:
key = (person_id, race_id)
edge_rows[key] = {
"person_id": person_id,
"race_id": race_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_column": "RACE",
"source_code": source_code,
"multrace": multrace,
}
def iter_source_batches(
year: int,
source_table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
header = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
wanted = {"ST_CASE", "VEH_NO", "PER_NO", "RACE"}
if "MULTRACE" in columns:
wanted.add("MULTRACE")
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
race_id = int_code(record.get("RACE"))
if race_id is None or race_id not in VALID_RACE_EDGE_IDS:
continue
person_id = person_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"), record.get("PER_NO"))
if not person_id.rsplit("_", 1)[-1]:
continue
multrace = int_code(record.get("MULTRACE")) if "MULTRACE" in record else None
add_race_edge(edge_rows, person_id, year, race_id, source_table, race_id, multrace)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path in find_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, chunk_size):
yield year, source_table, rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_race: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_race[int(row["race_id"])] += 1
print(f"Expected RACE_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Race id : {dict(sorted(rows_by_race.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT race_id IF NOT EXISTS FOR (r:Race) REQUIRE r.id IS UNIQUE",
":Race(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT person_person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.person_id IS UNIQUE",
":Person(person_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if rows:
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_RACE_NODES_QUERY, rows=RACE_NODE_ROWS).consume()
print("Ensured Race category nodes")
if args.ensure_constraints_only:
print(f"Race constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Race edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:RACE_OF]->() RETURN count(r) AS count").single()["count"]
print(f"Race edge load finished UTC: {utc_now()}")
print(f"Race edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_rows:,}")
print(f"Total relationships matched: {total_matched:,}")
print(f"RACE_OF relationship count : {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Vehicle Special Factor to Vehicle Relationship (46.613s)
Average edge import time per year: 1.208s across 1975-1981.
Average edge import time per year: 1.032s across 1982-2009.
Average edge import time per year: 0.688s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS VehicleSF-to-Vehicle relationships in Neo4j.
Sources:
* 1975-2009: vehicle-family VEH_CF1/VEH_CF2 columns from vehicle.csv and
vehnit.csv where present
* 2010-2019: vehicle.csv VEH_SC1/VEH_SC2 and parkwork.csv PVEH_SC1/PVEH_SC2
* 2020-2022: vehiclesf.csv VEHICLESF and pvehiclesf.csv PVEHICLESF
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:VehicleSF)-[:VEHICLE_SF_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
VEHICLESF_NODES: dict[int, str] = {
1: "Tires (Does Not Include Wheels, See Value 16)",
2: "Brake System",
3: "Steering System-Tie Rod, Kingpin, Ball Joint, etc.",
4: "Suspension-Springs, Shock Absorbers, MacPherson Struts, Axle Bearing, Control Arms, etc.",
5: "Power Train (Power Train/Engine, 2001-2009)-Universal Joint, Drive Shaft, Transmission, etc.",
6: "Exhaust System",
7: "Headlights",
8: "Signal Lights",
9: "Other Lights",
10: "Horn",
11: "Mirrors",
12: "Wipers",
13: "Driver Seating and Control",
14: "Body, Doors, Hood, Other",
15: "Trailer Hitch",
16: "Wheels",
17: "Air Bags",
18: "Other Vehicle Defects",
19: "Safety Belts",
29: "Default Code Used for Vehicle Numbering",
30: "Multi-Wheeled Motorcycle Conversion",
31: "Hit-and-Run Vehicle",
32: "Vehicle Registration for a Person with a Disability",
33: "Vehicle Being Pushed by Non-Motorist",
34: "Vehicle Impact Point-the Result of Something Set in Motion",
35: "Reconstructed/Altered Vehicle",
37: "Transporting Children to/From Head Start/Day Care",
38: "Vehicle Went Airborne During Crash",
39: "Highway Construction, Maintenance, or Utility Vehicle, In-Transport (Inside or Outside Work Zone)",
40: "Highway Incident Response Vehicle",
41: "Police, Fire, or EMS Vehicle Working at the Scene of an Emergency or Performing Other Traffic Control Activities",
42: "Other Working Vehicle (Not Construction, Maintenance, Utility, Police, Fire, or EMS Vehicle)",
43: "Hazardous Materials/Cargo Released From This Vehicle",
44: "Adaptive Equipment",
45: "Slide-in Camper",
99: "Unknown",
999: "Unknown",
}
VEHICLESF_NODE_ROWS = [{"id": code, "name": name} for code, name in VEHICLESF_NODES.items()]
VALID_VEHICLESF_IDS = set(VEHICLESF_NODES)
MERGE_VEHICLESF_NODES_QUERY = """
UNWIND $rows AS row
MERGE (sf:VehicleSF {id: row.id})
SET sf.name = row.name
RETURN count(sf) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (veh:Vehicle {vehicle_id: row.vehicle_id})
MATCH (sf:VehicleSF {id: row.vehiclesf_id})
MERGE (sf)-[r:VEHICLE_SF_OF]->(veh)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "vehiclesf_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS VehicleSF-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure VehicleSF constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def vehiclesf_ids_for(year: int, code: int) -> list[int]:
if code == 0:
return []
if 1975 <= year <= 1981 and code == 1:
return [1, 16]
if code in VALID_VEHICLESF_IDS:
return [code]
return []
def add_vehiclesf_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
vehiclesf_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, vehiclesf_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"vehiclesf_id": vehiclesf_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_legacy_source_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit))
return files
def find_vehicle_sc_source_files(base: Path) -> list[tuple[int, str, Path, tuple[str, str]]]:
files: list[tuple[int, str, Path, tuple[str, str]]] = []
for year in range(2010, 2020):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle, ("VEH_SC1", "VEH_SC2")))
parkwork = find_file(national, ["parkwork.csv", f"PARKWORK_{year}.CSV"])
if parkwork is not None:
files.append((year, "PARKWORK", parkwork, ("PVEH_SC1", "PVEH_SC2")))
return files
def find_vehiclesf_source_files(base: Path) -> list[tuple[int, str, Path, tuple[str, ...]]]:
files: list[tuple[int, str, Path, tuple[str, ...]]] = []
for year in range(2020, 2023):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehiclesf.csv", f"VEHICLESF_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLESF", vehicle, ("VEHICLESF",)))
pvehicle = find_file(national, ["pvehiclesf.csv", f"PVEHICLESF_{year}.CSV"])
if pvehicle is not None:
files.append((year, "PVEHICLESF", pvehicle, ("PVEHICLESF",)))
return files
def iter_source_batches(
year: int,
source_table: str,
csv_path: Path,
source_columns: tuple[str, ...],
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", *source_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in source_columns:
code = int_code(record.get(source_column))
if code is None:
continue
for vehiclesf_id in vehiclesf_ids_for(year, code):
add_vehiclesf_edge(edge_rows, vehicle_id, year, vehiclesf_id, source_table, source_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path in find_legacy_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, ("VEH_CF1", "VEH_CF2"), chunk_size):
yield year, source_table, rows
for year, source_table, csv_path, source_columns in find_vehicle_sc_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, source_columns, chunk_size):
yield year, source_table, rows
for year, source_table, csv_path, source_columns in find_vehiclesf_source_files(base):
for rows in iter_source_batches(year, source_table, csv_path, source_columns, chunk_size):
yield year, source_table, rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_vehiclesf: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_vehiclesf[int(row["vehiclesf_id"])] += 1
print(f"Expected VEHICLE_SF_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by VehicleSF id : {dict(sorted(rows_by_vehiclesf.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT vehiclesf_id IF NOT EXISTS FOR (sf:VehicleSF) REQUIRE sf.id IS UNIQUE",
":VehicleSF(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
result = session.run(MERGE_EDGE_QUERY, rows=rows)
matched = result.single()["matched"]
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
args = parse_args()
total_start = time.time()
print(f"Started UTC: {utc_now()}")
print(f"Base: {args.base}")
if args.dry_run:
dry_run(args.base, args.batch_size)
print(f"Finished UTC: {utc_now()}")
return
from neo4j import GraphDatabase
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
with driver.session(database=args.database) as session:
ensure_constraints(session)
node_result = session.run(MERGE_VEHICLESF_NODES_QUERY, rows=VEHICLESF_NODE_ROWS).single()
print(f"Ensured VehicleSF nodes: {node_result['matched']}")
if args.ensure_constraints_only:
return
rows_sent, rows_matched = import_relationships(session, args.base, args.batch_size)
total_elapsed = time.time() - total_start
print(f"\nTotal VEHICLE_SF_OF rows sent: {rows_sent:,}")
print(f"Total VEHICLE_SF_OF rows matched: {rows_matched:,}")
print(f"Total seconds: {total_elapsed:.3f}")
print(f"Total duration: {format_duration(total_elapsed)}")
finally:
driver.close()
print(f"Finished UTC: {utc_now()}")
if __name__ == "__main__":
main()
Violation to Vehicle Relationship (63.488s)
Average edge import time per year: 1.652s across 1975-1981.
Average edge import time per year: 1.317s across 1982-1996.
Average edge import time per year: 1.237s across 1997-2022.
python
#!/usr/bin/env python3
"""
Create FARS Violatn-to-Vehicle relationships in Neo4j.
Sources:
* 1975-1996: vehicle.csv VIOL_CHG
* 1997-2009: vehicle.csv and vehnit.csv VIOLCHG1/VIOLCHG2/VIOLCHG3
* 2010-2019: violatn.csv MVIOLATN
* 2020-2022: violatn.csv VIOLATION
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:Violatn)-[:COMMITTED_BY]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
VALID_VIOLATN_IDS = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 18, 19,
21, 22, 23, 24, 25, 26, 29,
31, 32, 33, 34, 35, 36, 37, 38, 39,
41, 42, 43, 45, 46, 48, 49,
51, 52, 53, 54, 55, 56, 58, 59,
61, 62, 63, 66, 67, 69,
71, 72, 73, 74, 75, 76, 79,
81, 82, 83, 84, 85, 86, 89,
91, 92, 93, 95, 97, 98, 99,
1001,
}
# 1975-1981 legacy values: 0 None, 1 Yes, 2 Pending, 9 Unknown.
MAP_1975_1981: dict[int, list[int]] = {
1: [1001],
9: [99],
}
# 1982-1996 legacy values. These are coarser than the 1997+ codes.
MAP_1982_1996: dict[int, list[int]] = {
1: [19], # Alcohol or Drugs -> general alcohol/drug/impairment violation.
2: [29], # Speeding -> speed-related violations generally.
3: [19, 29], # Alcohol or Drugs and Speeding.
4: [3], # Reckless Driving -> unsafe/reckless driving.
5: [71], # Driving with suspended/revoked license.
6: [98], # Other Moving Violation.
7: [79], # Non-Moving Violation.
8: [99], # Violation type unknown or other violation.
9: [99], # Unknown.
}
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (v:Vehicle {vehicle_id: row.vehicle_id})
MATCH (vn:Violatn {id: row.violatn_id})
MERGE (vn)-[r:COMMITTED_BY]->(v)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.created_from = "violatn_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Violatn-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Vehicle/Violatn constraints, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def map_violation_ids(year: int, code: int) -> list[int]:
if 1975 <= year <= 1981:
return MAP_1975_1981.get(code, [])
if 1982 <= year <= 1996:
return MAP_1982_1996.get(code, [])
if code == 0:
return []
if code in VALID_VIOLATN_IDS and code != 1001:
return [code]
return []
def add_edges(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
violatn_id: int,
source_table: str,
source_column: str,
source_code: int,
) -> None:
key = (vehicle_id, violatn_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"violatn_id": violatn_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def find_vehicle_source_files(base: Path) -> list[tuple[int, str, Path, list[str]]]:
files: list[tuple[int, str, Path, list[str]]] = []
for year in range(1975, 2010):
national = base / str(year) / "National"
columns = ["VIOL_CHG"] if year <= 1996 else ["VIOLCHG1", "VIOLCHG2", "VIOLCHG3"]
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle, columns))
if 1997 <= year <= 2009:
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit, columns))
return files
def find_violatn_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["violatn.csv", f"VIOLATN_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "VIOLATION" in columns:
files.append((year, path, "VIOLATION"))
elif "MVIOLATN" in columns:
files.append((year, path, "MVIOLATN"))
return files
def iter_vehicle_batches(
year: int,
source_table: str,
csv_path: Path,
violation_columns: list[str],
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", *violation_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in violation_columns:
code = int_code(record.get(source_column))
if code is None:
continue
for violatn_id in map_violation_ids(year, code):
add_edges(edge_rows, vehicle_id, year, violatn_id, source_table, source_column, code)
yield list(edge_rows.values())
def iter_violatn_batches(
year: int,
csv_path: Path,
violation_column: str,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", violation_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
code = int_code(record.get(violation_column))
if code is None:
continue
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for violatn_id in map_violation_ids(year, code):
add_edges(edge_rows, vehicle_id, year, violatn_id, "VIOLATN", violation_column, code)
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, source_table, csv_path, columns in find_vehicle_source_files(base):
for rows in iter_vehicle_batches(year, source_table, csv_path, columns, chunk_size):
yield year, source_table, rows
for year, csv_path, violation_column in find_violatn_files(base):
for rows in iter_violatn_batches(year, csv_path, violation_column, chunk_size):
yield year, "VIOLATN", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_violatn: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_violatn[int(row["violatn_id"])] += 1
print(f"Expected COMMITTED_BY relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Violatn id : {dict(sorted(rows_by_violatn.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT violatn_id IF NOT EXISTS FOR (vn:Violatn) REQUIRE vn.id IS UNIQUE",
":Violatn(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if rows:
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
if args.ensure_constraints_only:
print(f"Violatn constraints ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Violatn edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:COMMITTED_BY]->() RETURN count(r) AS count").single()["count"]
print(f"Violatn edge load finished UTC: {utc_now()}")
print(f"Violatn edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"COMMITTED_BY relationship count: {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Crash Vision-Related Factor to Vehicle Relationship (38.442s)
Average edge import time per year: 1.246s across 1975-1981.
Average edge import time per year: 0.787s across 1982-2009.
Average edge import time per year: 0.592s across 2010-2022.
python
#!/usr/bin/env python3
"""
Create FARS Vision-to-Vehicle relationships in Neo4j.
Sources:
* 1975-1981: accident.csv CF1/CF2/CF3, crash-level fallback linked to all
vehicles in the crash
* 1982-2008: vehicle-family DR_CF1/DR_CF2/DR_CF3/DR_CF4 legacy columns
vehicle.csv, and vehnit.csv where present
* 2009: vehicle-family D_VISION1/D_VISION2/D_VISION3 columns
* 2010-2019: vision.csv MVISOBSC
* 2020-2022: vision.csv VISION
The graph join uses vehicle_id = <year>_<ST_CASE>_<VEH_NO>. Raw ST_CASE and
VEH_NO are not globally unique across years.
Relationship:
(:Vision)-[:VISION_OF]->(:Vehicle)
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
VISION_NODES: dict[int, str] = {
1: "Rain, Snow, Fog, Smoke, Sand, Dust",
2: "Reflected Glare, Bright Sunlight, Headlights",
3: "Curve, Hill, or Other Roadway Design Features",
4: "Building, Billboard, or Other Structure",
5: "Trees, Crops, Vegetation",
6: "In-Transport Motor Vehicle (Including Load)",
7: "Not In-Transport Motor Vehicle (Parked, Working)",
8: "Splash or Spray of Passing Vehicle",
9: "Inadequate Defrost or Defog System",
10: "Inadequate Vehicle Lighting System",
11: "Obstructing Interior to the Vehicle",
12: "External Mirrors",
13: "Broken or Improperly Cleaned Windshield",
14: "Obstructing Angles on Vehicle",
95: "No Driver Present/Unknown if Driver Present",
97: "Vision Obscured - No Details",
98: "Other Visual Obstruction",
99: "Reported as Unknown",
1001: "Head Restraints",
}
VISION_NODE_ROWS = [
{"id": vision_id, "name": name}
for vision_id, name in VISION_NODES.items()
]
VALID_VISION_IDS = set(VISION_NODES)
MERGE_VISION_NODES_QUERY = """
UNWIND $rows AS row
MERGE (v:Vision {id: row.id})
SET v.name = row.name
RETURN count(v) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (veh:Vehicle {vehicle_id: row.vehicle_id})
MATCH (vis:Vision {id: row.vision_id})
MERGE (vis)-[r:VISION_OF]->(veh)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_columns = row.source_columns,
r.source_codes = row.source_codes,
r.source_scope = row.source_scope,
r.created_from = "vision_edges"
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Vision-to-Vehicle relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read sources and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Vision constraints/nodes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def vehicle_id_for(year: int, st_case: Any, veh_no: Any) -> str:
st_case_key = identifier_part(st_case)
if not st_case_key.startswith(f"{year}_"):
st_case_key = f"{year}_{st_case_key}"
return f"{st_case_key}_{identifier_part(veh_no)}"
def accident_id_for(year: int, st_case: Any) -> str:
st_case_key = identifier_part(st_case)
if st_case_key.startswith(f"{year}_"):
return st_case_key
return f"{year}_{st_case_key}"
def find_file(national: Path, names: list[str]) -> Path | None:
for name in names:
path = national / name
if path.exists():
return path
lower_names = {name.lower() for name in names}
for path in sorted(national.iterdir()) if national.exists() else []:
if path.is_file() and path.name.lower() in lower_names:
return path
return None
def map_accident_factor(code: int) -> int | None:
if 1 <= code <= 7:
return code
if code == 8:
return 98
return None
def map_legacy_driver_factor(year: int, code: int) -> int | None:
if not (1982 <= year <= 2008):
return None
mapping = {
61: 1,
62: 2,
63: 3,
64: 4,
65: 5,
66: 6,
67: 7,
68: 8,
69: 9,
70: 10,
71: 14,
72: 12,
75: 13,
76: 98,
}
if 1982 <= year <= 2001 and code == 73:
return 12
if 1982 <= year <= 2001 and code == 74:
return 1001
return mapping.get(code)
def add_vision_edge(
edge_rows: dict[tuple[str, int], dict[str, Any]],
vehicle_id: str,
year: int,
vision_id: int,
source_table: str,
source_column: str,
source_code: int,
source_scope: str,
) -> None:
key = (vehicle_id, vision_id)
row = edge_rows.setdefault(
key,
{
"vehicle_id": vehicle_id,
"vision_id": vision_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_columns": [],
"source_codes": [],
"source_scope": source_scope,
},
)
if source_column not in row["source_columns"]:
row["source_columns"].append(source_column)
if source_code not in row["source_codes"]:
row["source_codes"].append(source_code)
def vehicle_ids_by_accident(base: Path, year: int) -> dict[str, list[str]]:
national = base / str(year) / "National"
vehicle_path = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle_path is None:
return {}
df = pd.read_csv(vehicle_path, encoding="latin-1", low_memory=False, usecols=lambda c: normalize_column(c) in {"ST_CASE", "VEH_NO"})
df.columns = normalize_columns(df.columns)
mapping: dict[str, list[str]] = {}
for record in df.to_dict(orient="records"):
accident_id = accident_id_for(year, record.get("ST_CASE"))
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if vehicle_id.rsplit("_", 1)[-1]:
mapping.setdefault(accident_id, []).append(vehicle_id)
return mapping
def find_accident_source_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year in range(1975, 1982):
national = base / str(year) / "National"
path = find_file(national, ["accident.csv", f"ACCIDENT_{year}.CSV"])
if path is not None:
files.append((year, path))
return files
def find_legacy_source_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
for year in range(1982, 2009):
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit))
return files
def find_2009_vision_files(base: Path) -> list[tuple[int, str, Path]]:
files: list[tuple[int, str, Path]] = []
year = 2009
national = base / str(year) / "National"
vehicle = find_file(national, ["vehicle.csv", f"VEHICLE_{year}.CSV"])
if vehicle is not None:
files.append((year, "VEHICLE", vehicle))
vehnit = find_file(national, ["vehnit.csv", f"VEHNIT_{year}.CSV"])
if vehnit is not None:
files.append((year, "VEHNIT", vehnit))
return files
def find_vision_files(base: Path) -> list[tuple[int, Path, str]]:
files: list[tuple[int, Path, str]] = []
for year in range(2010, 2023):
national = base / str(year) / "National"
path = find_file(national, ["vision.csv", f"VISION_{year}.CSV"])
if path is None:
continue
header = pd.read_csv(path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
if "VISION" in columns:
files.append((year, path, "VISION"))
elif "MVISOBSC" in columns:
files.append((year, path, "MVISOBSC"))
return files
def iter_accident_batches(base: Path, year: int, csv_path: Path, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
accident_vehicles = vehicle_ids_by_accident(base, year)
wanted = {"ST_CASE", "CF1", "CF2", "CF3"}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_ids = accident_vehicles.get(accident_id_for(year, record.get("ST_CASE")), [])
if not vehicle_ids:
continue
for source_column in ("CF1", "CF2", "CF3"):
code = int_code(record.get(source_column))
if code is None:
continue
vision_id = map_accident_factor(code)
if vision_id is None:
continue
for vehicle_id in vehicle_ids:
add_vision_edge(edge_rows, vehicle_id, year, vision_id, "ACCIDENT", source_column, code, "crash")
yield list(edge_rows.values())
def iter_legacy_batches(
year: int,
source_table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
header = pd.read_csv(csv_path, nrows=0, encoding="latin-1")
columns = {normalize_column(column) for column in header.columns}
dr_columns = [column for column in ("DR_CF1", "DR_CF2", "DR_CF3", "DR_CF4") if column in columns]
wanted = {"ST_CASE", "VEH_NO", *dr_columns}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in dr_columns:
code = int_code(record.get(source_column))
if code is None:
continue
vision_id = map_legacy_driver_factor(year, code)
if vision_id is None:
continue
add_vision_edge(edge_rows, vehicle_id, year, vision_id, source_table, source_column, code, "vehicle")
yield list(edge_rows.values())
def iter_2009_batches(
year: int,
source_table: str,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", "D_VISION1", "D_VISION2", "D_VISION3"}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
for source_column in ("D_VISION1", "D_VISION2", "D_VISION3"):
code = int_code(record.get(source_column))
if code is None or code not in VALID_VISION_IDS:
continue
add_vision_edge(edge_rows, vehicle_id, year, code, source_table, source_column, code, "vehicle")
yield list(edge_rows.values())
def iter_vision_batches(year: int, csv_path: Path, source_column: str, chunk_size: int) -> Iterable[list[dict[str, Any]]]:
wanted = {"ST_CASE", "VEH_NO", source_column}
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in wanted,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
edge_rows: dict[tuple[str, int], dict[str, Any]] = {}
for record in chunk.to_dict(orient="records"):
code = int_code(record.get(source_column))
if code is None or code not in VALID_VISION_IDS:
continue
vehicle_id = vehicle_id_for(year, record.get("ST_CASE"), record.get("VEH_NO"))
if not vehicle_id.rsplit("_", 1)[-1]:
continue
add_vision_edge(edge_rows, vehicle_id, year, code, "VISION", source_column, code, "vehicle")
yield list(edge_rows.values())
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path in find_accident_source_files(base):
for rows in iter_accident_batches(base, year, csv_path, chunk_size):
yield year, "ACCIDENT", rows
for year, source_table, csv_path in find_legacy_source_files(base):
for rows in iter_legacy_batches(year, source_table, csv_path, chunk_size):
yield year, source_table, rows
for year, source_table, csv_path in find_2009_vision_files(base):
for rows in iter_2009_batches(year, source_table, csv_path, chunk_size):
yield year, source_table, rows
for year, csv_path, source_column in find_vision_files(base):
for rows in iter_vision_batches(year, csv_path, source_column, chunk_size):
yield year, "VISION", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_by_vision: Counter[int] = Counter()
total_rows = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total_rows += len(rows)
for row in rows:
rows_by_vision[int(row["vision_id"])] += 1
print(f"Expected VISION_OF relationships : {total_rows:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by Vision id : {dict(sorted(rows_by_vision.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT vision_id IF NOT EXISTS FOR (v:Vision) REQUIRE v.id IS UNIQUE",
":Vision(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT vehicle_vehicle_id IF NOT EXISTS FOR (v:Vehicle) REQUIRE v.vehicle_id IS UNIQUE",
":Vehicle(vehicle_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if rows:
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} edges in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_VISION_NODES_QUERY, rows=VISION_NODE_ROWS).consume()
print("Ensured Vision category nodes")
if args.ensure_constraints_only:
print(f"Vision constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Vision edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:VISION_OF]->() RETURN count(r) AS count").single()["count"]
print(f"Vision edge load finished UTC : {utc_now()}")
print(f"Vision edge load seconds : {load_elapsed:.3f}")
print(f"Total source edge rows : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"VISION_OF relationship count : {rel_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Weather to Accident Relationship (183.052s)
Average edge import time per year: 3.951s across 1975-2006.
Average edge import time per year: 3.596s across 2007-2019.
Average edge import time per year: 3.293s across 2020-2022.
python
#!/usr/bin/env python3
"""
Create FARS Accident-to-Weather relationships in Neo4j.
This script maps historical FARS weather encodings onto the normalized
:Weather nodes already used in the graph. It uses accident_id = <year>_<ST_CASE>
for all Accident joins; raw ST_CASE is not globally unique across years.
Source logic:
* 1975-2006: Accident.WEATHER
* 1975-1981: Accident.CF1/CF2/CF3 code 20 also indicates Severe Crosswinds
* 2007-2019: Accident.WEATHER1 and Accident.WEATHER2, because Accident.WEATHER
is a derived hierarchy and can hide a second condition
* 2020-2022: Weather data file, one row per atmospheric condition
Relationship:
(:Accident)-[:OCCURRED_IN]->(:Weather)
When one historical code maps ambiguously to multiple modern Weather nodes,
the relationship gets or_flag = 1. Non-ambiguous relationships do not carry
or_flag.
"""
from __future__ import annotations
import argparse
import math
import os
import sys
import time
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
DEFAULT_BASE = Path("/datasets/prop_datasets/fars")
DEFAULT_URI = "bolt://localhost:7689"
DEFAULT_USER = "neo4j"
DEFAULT_DATABASE = "fars"
DEFAULT_BATCH_SIZE = 10000
WEATHER_NODES: dict[int, str] = {
1: "Clear",
2: "Rain (Mist)",
3: "Sleet (Hail)",
4: "Snow",
6: "Severe Crosswinds",
7: "Blowing Sand, Soil, Dirt",
8: "Other",
9: "Cloudy",
11: "Blowing Snow",
12: "Freezing Rain or Drizzle",
98: "Not Reported",
99: "Unknown",
1001: "Fog",
1002: "Smog, Smoke",
}
WEATHER_NODE_ROWS = [
{"id": weather_id, "name": name}
for weather_id, name in WEATHER_NODES.items()
]
MERGE_WEATHER_NODES_QUERY = """
UNWIND $rows AS row
MERGE (w:Weather {id: row.id})
SET w.name = row.name
RETURN count(w) AS matched
"""
MERGE_EDGE_QUERY = """
UNWIND $rows AS row
MATCH (a:Accident {accident_id: row.accident_id})
MATCH (w:Weather {id: row.weather_id})
MERGE (a)-[r:OCCURRED_IN]->(w)
SET r.DATA_YEAR = row.DATA_YEAR,
r.source_table = row.source_table,
r.source_column = row.source_column,
r.source_code = row.source_code,
r.created_from = "weather_edges"
FOREACH (_ IN CASE WHEN row.or_flag = 1 THEN [1] ELSE [] END | SET r.or_flag = 1)
FOREACH (_ IN CASE WHEN row.or_flag = 1 THEN [] ELSE [1] END | REMOVE r.or_flag)
RETURN count(r) AS matched
"""
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create FARS Accident-to-Weather relationships in Neo4j.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--base", type=Path, default=Path(os.getenv("FARS_BASE", DEFAULT_BASE)))
parser.add_argument("--uri", default=os.getenv("NEO4J_URI", DEFAULT_URI))
parser.add_argument("--user", default=os.getenv("NEO4J_USER", DEFAULT_USER))
parser.add_argument("--password", default=os.getenv("NEO4J_PASSWORD", ""))
parser.add_argument("--database", default=os.getenv("NEO4J_DATABASE", DEFAULT_DATABASE))
parser.add_argument("--batch-size", type=int, default=int(os.getenv("BATCH_SIZE", DEFAULT_BATCH_SIZE)))
parser.add_argument("--dry-run", action="store_true", help="Read source files and print expected counts only.")
parser.add_argument(
"--ensure-constraints-only",
action="store_true",
help="Only ensure Weather constraints/indexes, then exit without importing relationships.",
)
return parser.parse_args()
def format_duration(seconds: float) -> str:
mins, secs = divmod(int(seconds), 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {secs}s"
if mins:
return f"{mins}m {secs}s"
return f"{secs}s"
def seconds_text(seconds: float) -> str:
return f"{seconds:.3f}s"
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def normalize_column(name: Any) -> str:
return str(name).replace("\ufeff", "").replace("", "").strip().upper()
def normalize_columns(columns: Iterable[Any]) -> list[str]:
normalized: list[str] = []
seen: dict[str, int] = {}
for column in columns:
name = normalize_column(column)
if name in seen:
seen[name] += 1
name = f"{name}_{seen[name]}"
else:
seen[name] = 0
normalized.append(name)
return normalized
def identifier_part(value: Any) -> str:
if value is None or pd.isna(value):
return ""
if isinstance(value, str):
text = value.strip()
if text.endswith(".0") and text[:-2].isdigit():
return text[:-2]
return text
if isinstance(value, float) and math.isfinite(value) and value.is_integer():
return str(int(value))
if hasattr(value, "item"):
return identifier_part(value.item())
return str(value).strip()
def int_code(value: Any) -> int | None:
text = identifier_part(value)
if not text:
return None
try:
return int(text)
except ValueError:
return None
def mapping_for(year: int, code: int, source_column: str) -> list[tuple[int, int | None]]:
if 1975 <= year <= 1979:
return {
1: [(1, None)],
2: [(2, None)],
3: [(3, None)],
4: [(4, None)],
7: [(9, None)],
9: [(99, None)],
}.get(code, [])
if 1980 <= year <= 1981:
return {
1: [(1, None)],
2: [(2, None)],
3: [(3, None)],
4: [(4, None)],
5: [(1001, None)],
8: [(1002, 1), (7, 1)],
9: [(99, None)],
}.get(code, [])
if 1982 <= year <= 2006:
return {
1: [(1, None)],
2: [(2, None)],
3: [(3, None)],
4: [(4, None)],
5: [(1001, None)],
6: [(2, None), (1001, None)],
7: [(3, None), (1001, None)],
8: [(1002, 1), (7, 1), (8, 1)],
9: [(99, None)],
}.get(code, [])
if 2007 <= year <= 2009:
if code == 0 and source_column == "WEATHER2":
return []
return {
0: [(1, None)],
1: [(1, 1), (9, 1)],
2: [(2, None)],
3: [(3, None)],
4: [(4, 1), (11, 1)],
5: [(1001, 1), (1002, 1)],
6: [(6, None)],
7: [(7, None)],
8: [(8, None)],
9: [(99, None)],
}.get(code, [])
if 2010 <= year <= 2012:
if code == 0:
return []
return {
1: [(1, None)],
2: [(2, None)],
3: [(3, 1), (12, 1)],
4: [(4, None)],
5: [(1001, 1), (1002, 1)],
6: [(6, None)],
7: [(7, None)],
8: [(8, None)],
10: [(9, None)],
11: [(11, None)],
98: [(98, None)],
99: [(99, None)],
}.get(code, [])
if 2013 <= year <= 2022:
if code == 0:
return []
return {
1: [(1, None)],
2: [(2, None)],
3: [(3, None)],
4: [(4, None)],
5: [(1001, 1), (1002, 1)],
6: [(6, None)],
7: [(7, None)],
8: [(8, None)],
10: [(9, None)],
11: [(11, None)],
12: [(12, None)],
98: [(98, None)],
99: [(99, None)],
}.get(code, [])
return []
def find_accident_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if 1975 <= year <= 2019:
csv_path = year_dir / "National" / "accident.csv"
if csv_path.exists():
files.append((year, csv_path))
return files
def find_weather_files(base: Path) -> list[tuple[int, Path]]:
files: list[tuple[int, Path]] = []
for year_dir in sorted(base.iterdir()):
if not year_dir.is_dir() or not year_dir.name.isdigit():
continue
year = int(year_dir.name)
if 2020 <= year <= 2022:
csv_path = year_dir / "National" / "weather.csv"
if csv_path.exists():
files.append((year, csv_path))
return files
def relationship_rows_from_code(
year: int,
st_case: Any,
source_table: str,
source_column: str,
source_code: Any,
) -> list[dict[str, Any]]:
code = int_code(source_code)
if code is None:
return []
accident_id = f"{year}_{identifier_part(st_case)}"
rows = []
for weather_id, or_flag in mapping_for(year, code, source_column):
rows.append(
{
"accident_id": accident_id,
"weather_id": weather_id,
"DATA_YEAR": year,
"source_table": source_table,
"source_column": source_column,
"source_code": code,
"or_flag": or_flag,
}
)
return rows
def iter_accident_weather_rows(
year: int,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
needed = {"ST_CASE"}
if year <= 2006:
needed.add("WEATHER")
else:
needed.update({"WEATHER1", "WEATHER2"})
if 1975 <= year <= 1981:
needed.update({"CF1", "CF2", "CF3"})
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in needed,
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
rows: list[dict[str, Any]] = []
source_columns = ["WEATHER"] if year <= 2006 else ["WEATHER1", "WEATHER2"]
for record in chunk.to_dict(orient="records"):
for source_column in source_columns:
if source_column in record:
rows.extend(
relationship_rows_from_code(
year,
record["ST_CASE"],
"ACCIDENT",
source_column,
record.get(source_column),
)
)
if 1975 <= year <= 1981:
for cf_column in ("CF1", "CF2", "CF3"):
if int_code(record.get(cf_column)) == 20:
rows.append(
{
"accident_id": f"{year}_{identifier_part(record['ST_CASE'])}",
"weather_id": 6,
"DATA_YEAR": year,
"source_table": "ACCIDENT",
"source_column": cf_column,
"source_code": 20,
"or_flag": 1,
}
)
break
yield rows
def iter_table_weather_rows(
year: int,
csv_path: Path,
chunk_size: int,
) -> Iterable[list[dict[str, Any]]]:
chunks = pd.read_csv(
csv_path,
encoding="latin-1",
low_memory=False,
chunksize=chunk_size,
usecols=lambda c: normalize_column(c) in {"ST_CASE", "WEATHER"},
)
for chunk in chunks:
chunk.columns = normalize_columns(chunk.columns)
rows: list[dict[str, Any]] = []
for record in chunk.to_dict(orient="records"):
rows.extend(
relationship_rows_from_code(
year,
record["ST_CASE"],
"WEATHER",
"WEATHER",
record.get("WEATHER"),
)
)
yield rows
def iter_all_batches(base: Path, chunk_size: int) -> Iterable[tuple[int, str, list[dict[str, Any]]]]:
for year, csv_path in find_accident_files(base):
for rows in iter_accident_weather_rows(year, csv_path, chunk_size):
yield year, "ACCIDENT", rows
for year, csv_path in find_weather_files(base):
for rows in iter_table_weather_rows(year, csv_path, chunk_size):
yield year, "WEATHER", rows
def dry_run(base: Path, chunk_size: int) -> None:
start = time.time()
rows_by_year: Counter[int] = Counter()
rows_by_weather: Counter[int] = Counter()
rows_by_source: Counter[str] = Counter()
rows_with_or_flag = 0
total = 0
for year, source, rows in iter_all_batches(base, chunk_size):
rows_by_year[year] += len(rows)
rows_by_source[source] += len(rows)
total += len(rows)
for row in rows:
rows_by_weather[int(row["weather_id"])] += 1
if row["or_flag"] == 1:
rows_with_or_flag += 1
print(f"Expected OCCURRED_IN relationships: {total:,}")
print(f"Rows with or_flag=1 : {rows_with_or_flag:,}")
print(f"Source edge counts : {dict(sorted(rows_by_source.items()))}")
print("Rows by year:")
for year in sorted(rows_by_year):
print(f" {year}: {rows_by_year[year]:,}")
print(f"Rows by weather id : {dict(sorted(rows_by_weather.items()))}")
print(f"Dry run seconds : {time.time() - start:.3f}")
def run_ddl(session: Any, query: str, label: str) -> None:
try:
session.run(query).consume()
print(f"Ensured {label}")
except Exception as exc:
print(f"WARNING: could not ensure {label}: {exc}")
def ensure_constraints(session: Any) -> None:
run_ddl(
session,
"CREATE CONSTRAINT weather_id IF NOT EXISTS FOR (w:Weather) REQUIRE w.id IS UNIQUE",
":Weather(id) unique constraint",
)
run_ddl(
session,
"CREATE CONSTRAINT accident_id IF NOT EXISTS FOR (a:Accident) REQUIRE a.accident_id IS UNIQUE",
":Accident(accident_id) unique constraint",
)
def import_relationships(session: Any, base: Path, batch_size: int) -> tuple[int, int]:
total_rows = 0
total_matched = 0
current_year: int | None = None
year_rows = 0
year_start = time.time()
for year, _, rows in iter_all_batches(base, batch_size):
if current_year is None:
current_year = year
if year != current_year:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} rows in {seconds_text(elapsed)} ({format_duration(elapsed)})")
current_year = year
year_rows = 0
year_start = time.time()
if not rows:
continue
record = session.run(MERGE_EDGE_QUERY, rows=rows).single()
matched = int(record["matched"]) if record else 0
total_rows += len(rows)
total_matched += matched
year_rows += len(rows)
if current_year is not None:
elapsed = time.time() - year_start
print(f" Done {current_year}: {year_rows:,} rows in {seconds_text(elapsed)} ({format_duration(elapsed)})")
return total_rows, total_matched
def main() -> None:
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True)
args = parse_args()
if args.dry_run:
dry_run(args.base, args.batch_size)
return
try:
from neo4j import GraphDatabase
except ModuleNotFoundError as exc:
raise SystemExit(
"Missing Python package 'neo4j'. Install it with: "
"python -m pip install --user neo4j"
) from exc
total_start = time.time()
driver = GraphDatabase.driver(args.uri, auth=(args.user, args.password))
try:
driver.verify_connectivity()
with driver.session(database=args.database) as session:
ensure_constraints(session)
session.run(MERGE_WEATHER_NODES_QUERY, rows=WEATHER_NODE_ROWS).consume()
print("Ensured Weather category nodes")
if args.ensure_constraints_only:
print(f"Weather constraints/nodes ensured on database {args.database!r}")
return
print(f"Neo4j URI : {args.uri}")
print(f"Neo4j database : {args.database}")
print(f"Batch size : {args.batch_size}")
print(f"Script started UTC : {utc_now()}")
print("")
load_start = time.time()
print(f"Weather edge load started UTC: {utc_now()}")
total_rows, total_matched = import_relationships(session, args.base, args.batch_size)
load_elapsed = time.time() - load_start
rel_count = session.run("MATCH ()-[r:OCCURRED_IN]->() RETURN count(r) AS count").single()["count"]
or_count = session.run("MATCH ()-[r:OCCURRED_IN]->() WHERE r.or_flag = 1 RETURN count(r) AS count").single()["count"]
print(f"Weather edge load finished UTC: {utc_now()}")
print(f"Weather edge load seconds : {load_elapsed:.3f}")
print(f"Total source rows processed : {total_rows:,}")
print(f"Total relationships matched : {total_matched:,}")
print(f"OCCURRED_IN relationship count: {rel_count:,}")
print(f"OCCURRED_IN or_flag count : {or_count:,}")
print(f"Total script seconds : {time.time() - total_start:.3f}")
finally:
driver.close()
if __name__ == "__main__":
main()
Queries
Select a query below to view its logic and description.
1. Neighborhood
2. Path
3. Pattern
4. Aggregation
5. Analytics
6. Hybrid
7. Others
Q1
TBD
Q2
TBD
Q3
TBD
Q4
TBD
Q5
TBD
Q6
TBD
Q7
TBD
Q8
TBD
Q9
TBD
Q10
TBD
Q11
TBD
Q12
TBD
Q13
TBD
Q14
TBD
Q15
TBD
Q16
TBD
Q17
TBD
Q18
TBD
Q19
TBD
Q20
TBD
Q21
TBD
Q22
TBD
Q23
TBD
Q24
TBD
Q25
TBD
Q26
TBD
Q27
TBD
Q28
TBD
Q29
TBD
Q30
TBD
Q31
TBD
Q32
TBD
Q33
TBD