diff --git a/Cargo.lock b/Cargo.lock index 603df94772..12418b652c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,9 +124,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.41" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "approx" @@ -713,6 +713,7 @@ dependencies = [ "kml", "log", "osm2streets", + "popgetter", "raw_map", "serde", "streets_reader", @@ -1509,9 +1510,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "2.6.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ebd3504ad6116843b8375ad70df74e7bfe83cac77a1f3fe73200c844d43bfe0" +checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" [[package]] name = "futures" @@ -2765,6 +2766,7 @@ dependencies = [ "md5", "osm2streets", "petgraph", + "popgetter", "rand", "rand_xorshift", "raw_map", @@ -3427,6 +3429,20 @@ dependencies = [ "synthpop", ] +[[package]] +name = "popgetter" +version = "0.1.0" +source = "git+https://github.com/dabreegster/popgetter/#20eb9d3facae9f0f01394fd5493a0adfbc8578f5" +dependencies = [ + "anyhow", + "fs-err", + "geo", + "geo-types", + "geojson", + "serde", + "topojson", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -3591,6 +3607,7 @@ dependencies = [ "abstutil", "geom", "osm2streets", + "popgetter", "serde", "strum", "strum_macros", @@ -4591,6 +4608,17 @@ dependencies = [ "serde", ] +[[package]] +name = "topojson" +version = "0.5.1" +source = "git+https://github.com/dabreegster/topojson?branch=update#f962c7b3ae372925a74c327d90ee73ff487d8a6b" +dependencies = [ + "geojson", + "log", + "serde", + "serde_json", +] + [[package]] name = "tower-service" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 5a54f3f27e..94a91180ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,12 +37,12 @@ opt-level = 3 # Specify the versions for common dependencies just once here, instead of # repeating in a bunch of crates [workspace.dependencies] -anyhow = "1.0.38" +anyhow = "1.0.69" bincode = "1.3.1" colorous = "1.0.9" contour = "0.7.0" csv = "1.2.0" -fs-err = "2.6.0" +fs-err = "2.9.0" geo = "0.24.0" geojson = { version = "0.24.0", features = ["geo-types"] } getrandom = "0.2.3" diff --git a/convert_osm/Cargo.toml b/convert_osm/Cargo.toml index d4ad05d80e..6185c18417 100644 --- a/convert_osm/Cargo.toml +++ b/convert_osm/Cargo.toml @@ -14,6 +14,7 @@ geom = { path = "../geom" } kml = { path = "../kml" } log = { workspace = true } osm2streets = { git = "https://github.com/a-b-street/osm2streets" } +popgetter = { git = "https://github.com/dabreegster/popgetter/" } raw_map = { path = "../raw_map" } serde = { workspace = true } streets_reader = { git = "https://github.com/a-b-street/osm2streets" } diff --git a/convert_osm/src/lib.rs b/convert_osm/src/lib.rs index b475b6d59a..09517f95ad 100644 --- a/convert_osm/src/lib.rs +++ b/convert_osm/src/lib.rs @@ -152,6 +152,12 @@ pub fn convert( gtfs::import(&mut map).unwrap(); } + timer.start("Add census data"); + if let Err(err) = add_census(&mut map) { + error!("Skipping census data: {err}"); + } + timer.stop("Add census data"); + if map.name == MapName::new("gb", "bristol", "east") { bristol_hack(&mut map); } @@ -379,3 +385,23 @@ fn filter_crosswalks( } } } + +fn add_census(map: &mut RawMap) -> Result<()> { + // TODO Fixed to one area right now. Assumes the file exists. + let input_path = "data/input/shared/popgetter/england.topojson"; + let boundary = map + .streets + .boundary_polygon + .to_geo_wgs84(&map.streets.gps_bounds); + for (geo_polygon, census_zone) in popgetter::clip_zones(input_path, boundary)? { + match Polygon::from_geo_wgs84(geo_polygon, &map.streets.gps_bounds) { + Ok(polygon) => { + map.census_zones.push((polygon, census_zone)); + } + Err(err) => { + warn!("Skipping census zone {}: {}", census_zone.id, err); + } + } + } + Ok(()) +} diff --git a/data/MANIFEST.json b/data/MANIFEST.json index 777d771c0e..03e5c52740 100644 --- a/data/MANIFEST.json +++ b/data/MANIFEST.json @@ -216,8 +216,8 @@ "compressed_size_bytes": 13642164 }, "data/input/fr/paris/raw_maps/center.bin": { - "checksum": "6f2811ae79e8b71cb2f596fb456d7dfe", - "uncompressed_size_bytes": 25065354, + "checksum": "896d1a1ae5a7dacd63d32b5fd6d931f6", + "uncompressed_size_bytes": 25065338, "compressed_size_bytes": 6777329 }, "data/input/fr/paris/raw_maps/east.bin": { @@ -1530,6 +1530,11 @@ "uncompressed_size_bytes": 272419737, "compressed_size_bytes": 271888733 }, + "data/input/shared/popgetter/england.topojson": { + "checksum": "a894795a5bfadf60899f4ba892a32789", + "uncompressed_size_bytes": 52802533, + "compressed_size_bytes": 7976809 + }, "data/input/shared/wu03ew_v2.csv": { "checksum": "6614880a64e448b4e9a7255ab5355d00", "uncompressed_size_bytes": 108977615, diff --git a/geom/src/polygon.rs b/geom/src/polygon.rs index 104fb4d41f..44c82c6172 100644 --- a/geom/src/polygon.rs +++ b/geom/src/polygon.rs @@ -2,7 +2,9 @@ use std::collections::BTreeMap; use std::fmt; use anyhow::Result; -use geo::{Area, BooleanOps, Contains, ConvexHull, Intersects, SimplifyVwPreserve}; +use geo::{ + Area, BooleanOps, Contains, ConvexHull, Intersects, MapCoordsInPlace, SimplifyVwPreserve, +}; use serde::{Deserialize, Serialize}; use crate::{ @@ -410,8 +412,6 @@ impl Polygon { /// Optionally map the world-space points back to GPS. pub fn to_geojson(&self, gps: Option<&GPSBounds>) -> geojson::Geometry { - use geo::MapCoordsInPlace; - let mut geom: geo::Geometry = self.to_geo().into(); if let Some(ref gps_bounds) = gps { geom.map_coords_in_place(|c| { @@ -498,6 +498,24 @@ impl Polygon { fn to_geo(&self) -> geo::Polygon { self.clone().into() } + + /// Convert to `geo` and also map from world-space to WGS84 + pub fn to_geo_wgs84(&self, gps_bounds: &GPSBounds) -> geo::Polygon { + let mut p = self.to_geo(); + p.map_coords_in_place(|c| { + let gps = Pt2D::new(c.x, c.y).to_gps(gps_bounds); + (gps.x(), gps.y()).into() + }); + p + } + + pub fn from_geo_wgs84(mut polygon: geo::Polygon, gps_bounds: &GPSBounds) -> Result { + polygon.map_coords_in_place(|c| { + let pt = LonLat::new(c.x, c.y).to_pt(gps_bounds); + (pt.x(), pt.y()).into() + }); + polygon.try_into() + } } impl fmt::Display for Polygon { diff --git a/map_model/Cargo.toml b/map_model/Cargo.toml index 1d991dcb44..1912819e8f 100644 --- a/map_model/Cargo.toml +++ b/map_model/Cargo.toml @@ -17,6 +17,7 @@ log = { workspace = true } lyon = "1.0.1" md5 = "0.7.0" petgraph = { version = "0.6.3", features=["serde-1"] } +popgetter = { git = "https://github.com/dabreegster/popgetter/" } rand = { workspace = true } rand_xorshift = { workspace = true } raw_map = { path = "../raw_map" } diff --git a/map_model/src/lib.rs b/map_model/src/lib.rs index 4f4a3ef345..5278046e3c 100644 --- a/map_model/src/lib.rs +++ b/map_model/src/lib.rs @@ -30,6 +30,7 @@ extern crate log; use std::collections::BTreeMap; +use popgetter::CensusZone; use serde::{Deserialize, Serialize}; use abstio::MapName; @@ -115,6 +116,7 @@ pub struct Map { routing_params: RoutingParams, // Not the source of truth, just cached. zones: Vec, + census_zones: Vec<(Polygon, CensusZone)>, name: MapName, diff --git a/map_model/src/make/mod.rs b/map_model/src/make/mod.rs index b5a9df87c9..7a09f51af0 100644 --- a/map_model/src/make/mod.rs +++ b/map_model/src/make/mod.rs @@ -54,6 +54,7 @@ impl Map { areas: Vec::new(), parking_lots: Vec::new(), zones: Vec::new(), + census_zones: raw.census_zones.clone(), boundary_polygon: raw.streets.boundary_polygon.clone(), stop_signs: BTreeMap::new(), traffic_signals: BTreeMap::new(), diff --git a/map_model/src/map.rs b/map_model/src/map.rs index e9ba1bbff3..605a699860 100644 --- a/map_model/src/map.rs +++ b/map_model/src/map.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use anyhow::Result; use petgraph::graphmap::{DiGraphMap, UnGraphMap}; +use popgetter::CensusZone; use abstio::{CityName, MapName}; use abstutil::{prettyprint_usize, serialized_size_bytes, MultiMap, Tags, Timer}; @@ -104,6 +105,11 @@ impl Map { self.zones.len(), serialized_size_bytes(&self.zones), ), + ( + "census_zones", + self.census_zones.len(), + serialized_size_bytes(&self.census_zones), + ), ("pathfinder", 1, serialized_size_bytes(&self.pathfinder)), ]; costs.sort_by_key(|(_, _, bytes)| *bytes); @@ -135,6 +141,7 @@ impl Map { areas: Vec::new(), parking_lots: Vec::new(), zones: Vec::new(), + census_zones: Vec::new(), boundary_polygon: Ring::must_new(vec![ Pt2D::new(0.0, 0.0), Pt2D::new(1.0, 0.0), @@ -257,6 +264,10 @@ impl Map { &self.zones } + pub fn all_census_zones(&self) -> &Vec<(Polygon, CensusZone)> { + &self.census_zones + } + pub fn maybe_get_r(&self, id: RoadID) -> Option<&Road> { self.roads.get(id.0) } diff --git a/map_model/src/pathfind/mod.rs b/map_model/src/pathfind/mod.rs index 4c5290bfc1..1f1a24ae94 100644 --- a/map_model/src/pathfind/mod.rs +++ b/map_model/src/pathfind/mod.rs @@ -193,8 +193,6 @@ pub struct RoutingParams { /// TODO The route may cross one of these roads if it's the start or end! pub avoid_roads: BTreeSet, /// Related to `avoid_roads`, but used as an optimization in map construction - // TODO Serialize this normally. In the meantime, safe because the default is indeed empty. - #[serde(skip_serializing, skip_deserializing)] pub only_use_roads: BTreeSet, /// Don't allow movements between these roads at all. Only affects vehicle routing, not diff --git a/raw_map/Cargo.toml b/raw_map/Cargo.toml index b648c42fa1..5c00ce7079 100644 --- a/raw_map/Cargo.toml +++ b/raw_map/Cargo.toml @@ -10,5 +10,6 @@ abstutil = { path = "../abstutil" } geom = { path = "../geom" } serde = { workspace = true } osm2streets = { git = "https://github.com/a-b-street/osm2streets" } +popgetter = { git = "https://github.com/dabreegster/popgetter/" } strum = "0.24.1" strum_macros = "0.24.3" diff --git a/raw_map/src/lib.rs b/raw_map/src/lib.rs index c523f92419..c16c4d8bab 100644 --- a/raw_map/src/lib.rs +++ b/raw_map/src/lib.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use osm2streets::{osm, IntersectionID, RoadID, StreetNetwork}; +use popgetter::CensusZone; use serde::{Deserialize, Serialize}; use abstio::{CityName, MapName}; @@ -31,6 +32,7 @@ pub struct RawMap { pub parking_lots: Vec, pub parking_aisles: Vec<(osm::WayID, Vec)>, pub transit_routes: Vec, + pub census_zones: Vec<(Polygon, CensusZone)>, #[serde( serialize_with = "serialize_btreemap", deserialize_with = "deserialize_btreemap" @@ -74,6 +76,7 @@ impl RawMap { parking_lots: Vec::new(), parking_aisles: Vec::new(), transit_routes: Vec::new(), + census_zones: Vec::new(), transit_stops: BTreeMap::new(), bus_routes_on_roads: MultiMap::new(), osm_tags: BTreeMap::new(),