diff --git a/django/apps/aggregated/management/commands/update_aggregated_data.py b/django/apps/aggregated/management/commands/update_aggregated_data.py index be8b82989..dca5896a4 100644 --- a/django/apps/aggregated/management/commands/update_aggregated_data.py +++ b/django/apps/aggregated/management/commands/update_aggregated_data.py @@ -55,6 +55,7 @@ WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2 -- FOOTPRINT: Not calculated right now WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1 + WHEN P.project_type = {Project.Type.STREET.value} THEN 65 ELSE 1 END ) * COUNT(*) as time_spent_max_allowed @@ -110,6 +111,7 @@ WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2 -- FOOTPRINT: Not calculated right now WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1 + WHEN P.project_type = {Project.Type.STREET.value} THEN 65 ELSE 1 END ) * COUNT(*) as time_spent_max_allowed diff --git a/docs/source/_static/img/mapswipe-time-calculation.png b/docs/source/_static/img/mapswipe-time-calculation.png new file mode 100644 index 000000000..ebef8f83b Binary files /dev/null and b/docs/source/_static/img/mapswipe-time-calculation.png differ diff --git a/docs/source/diagrams.md b/docs/source/diagrams.md index 8ad2ccc56..8ac15c8a9 100644 --- a/docs/source/diagrams.md +++ b/docs/source/diagrams.md @@ -9,29 +9,56 @@ The Diagrams are drawn using [draw.io](https://.wwww.draw.io). You can download --- **Deployment Diagram:** -![Deployment Diagram](/_static/img/deployment_diagram.png) +![Deployment Diagram](_static/img/deployment_diagram.png) --- **Proposed Data Structure Project Type 1 - Firebase:** -![Data Structure - Firebase](/_static/img/data_structure-firebase-1.svg) +![Data Structure - Firebase](_static/img/data_structure-firebase-1.svg) --- **Proposed Data Structure Project Type 2 - Firebase:** -![Data Structure - Firebase](/_static/img/data_structure-firebase-2.svg) +![Data Structure - Firebase](_static/img/data_structure-firebase-2.svg) --- **Database Scheme - Postgres:** -![Database Schema - Postgres](/_static/img/database_schema-postgres.png) +![Database Schema - Postgres](_static/img/database_schema-postgres.png) --- **Entity Relationship Diagram - Postgres:** -![Entity Relationship Diagram- Postgres](/_static/img/entity_relationship_diagram-postgres.png) +![Entity Relationship Diagram- Postgres](_static/img/entity_relationship_diagram-postgres.png) --- **Database Schema - Analytics:** -![Database Schema - Analytics](/_static/img/database_schema-analytics.png) +![Database Schema - Analytics](_static/img/database_schema-analytics.png) + +--- + +**Mapping Sessions - Time Calculation** + +The diagram below is a visual representation of how time is calculated in MapSwipe. + +Step 1: User Mapping Session **sends data** to Firebase +- When a user completes a mapping session in the mobile/web app, the session payload (including start/end timestamps, user ID, session metadata, etc.) is sent in real time to Firebase. + +Step 2: Cron job **fetches data** from the firebase +- Every 3 minutes, a cron job syncs data for any new session records and pulls them into the backend. + +Step 3: Cron job **saves raw data** to Postgres database +- The cron job sends new session data to the Postgres database. + +Step 4: Cron job **reads raw data** from Postgres database +- Another cron job reads the raw data from Postgres database. + +Step 5: Cron job **saves aggregates** to Postgres database +- The cron job aggregates previous 24 hours data (end date - start date), sends back, and saves processed aggregated data to the Postgres database. + +Step 6: Community dashboard **queries aggregate data** from Postgres database +- The Community dashboard pulls the processed data from the Postgres database and updates the dashbaord with up-to-date stats. + + +![MapSwipe Time Calculation](_static/img/mapswipe-time-calculation.png) diff --git a/firebase/functions/src/index.ts b/firebase/functions/src/index.ts index 02a70dd37..793ed9c71 100644 --- a/firebase/functions/src/index.ts +++ b/firebase/functions/src/index.ts @@ -42,23 +42,46 @@ exports.groupUsersCounter = functions.database.ref('/v2/results/{projectId}/{gro const thisResultRef = admin.database().ref('/v2/results/' + context.params.projectId + '/' + context.params.groupId + '/' + context.params.userId ); const userGroupsRef = admin.database().ref('/v2/userGroups/'); + let appVersionString: string | undefined | null = undefined; + + type Args = Record + // eslint-disable-next-line require-jsdoc + function logger(message: string, extraArgs: Args = {}, logFunction: (typeof console.log) = console.log) { + const ctx: Args = { + message: message, + ...extraArgs, + project: context.params.projectId, + user: context.params.userId, + group: context.params.groupId, + version: appVersionString, + }; + const items = Object.keys(ctx).reduce( + (acc, key) => { + const value = ctx[key]; + if (value === undefined || value === null || value === '') { + return acc; + } + const item = `${key}[${value}]`; + return [...acc, item]; + }, + [] + ); + logFunction(items.join(' ')); + } // Check for specific user ids which have been identified as problematic. // These users have repeatedly uploaded harmful results. // Add new user ids to this list if needed. const userIds: string[] = []; - if ( userIds.includes(context.params.userId) ) { - console.log('suspicious user: ' + context.params.userId); - console.log('will remove this result and not update counters'); + if (userIds.includes(context.params.userId) ) { + console.log('Result removed because of suspicious user activity'); return thisResultRef.remove(); } const result = snapshot.val(); - - // New versions of app will have the appVersion defined (> 2.2.5) // appVersion: 2.2.5 (14)-dev - const appVersionString = result.appVersion as string | undefined | null; + appVersionString = result.appVersion; // Check if the app is of older version // (no need to check for specific version since old app won't sent the version info) @@ -68,11 +91,11 @@ exports.groupUsersCounter = functions.database.ref('/v2/results/{projectId}/{gro if (dataSnapshot.exists()) { const project = dataSnapshot.val(); - // Check if project type is validate and also has + // Check if project type is 'validate' and also has // custom options (i.e. these are new type of projects) if (project.projectType === 2 && project.customOptions) { // We remove the results submitted from older version of app (< v2.2.6) - console.info(`Result submitted for ${context.params.projectId} was discarded: submitted from older version of app`); + logger('Result removed because it was submitted from an older version', undefined, console.error); return thisResultRef.remove(); } } @@ -81,16 +104,13 @@ exports.groupUsersCounter = functions.database.ref('/v2/results/{projectId}/{gro // if result ref does not contain all required attributes we don't updated counters // e.g. due to some error when uploading from client if (!Object.prototype.hasOwnProperty.call(result, 'results')) { - console.log('no results attribute for ' + snapshot.ref); - console.log('will not update counters'); + logger('Not updating counters because results attribute was not found.', { result: String(snapshot.ref) }, console.error); return null; } else if (!Object.prototype.hasOwnProperty.call(result, 'endTime')) { - console.log('no endTime attribute for ' + snapshot.ref); - console.log('will not update counters'); + logger('Not updating counters because endTime attribute was not found.', { result: String(snapshot.ref) }, console.error); return null; } else if (!Object.prototype.hasOwnProperty.call(result, 'startTime')) { - console.log('no startTime attribute for ' + snapshot.ref); - console.log('will not update counters'); + logger('Not updating counters because startTime attribute was not found.', { result: String(snapshot.ref) }, console.error); return null; } @@ -103,8 +123,7 @@ exports.groupUsersCounter = functions.database.ref('/v2/results/{projectId}/{gro const mappingSpeed = (endTime - startTime) / numberOfTasks; if (mappingSpeed < 0.125) { // this about 8-times faster than the average time needed per task - console.log('unlikely high mapping speed: ' + mappingSpeed); - console.log('will remove this result and not update counters'); + logger('Result removed because of unlikely high mapping speed', { mappingSpeed: mappingSpeed }, console.warn); return thisResultRef.remove(); } @@ -117,10 +136,12 @@ exports.groupUsersCounter = functions.database.ref('/v2/results/{projectId}/{gro */ const dataSnapshot = await groupUsersRef.child(context.params.userId).once('value'); if (dataSnapshot.exists()) { - console.log('group contribution exists already. user: '+context.params.userId+' project: '+context.params.projectId+' group: '+context.params.groupId); + logger('Group contribution already exists.'); return null; } + // Update contributions + const latestNumberOfTasks = Object.keys(result['results']).length; await Promise.all([ userContributionRef.child(context.params.groupId).set(true), @@ -136,8 +157,8 @@ exports.groupUsersCounter = functions.database.ref('/v2/results/{projectId}/{gro }), ]); - // Tag userGroups of the user in the result + const userGroupsOfTheUserSnapshot = await userRef.child('userGroups').once('value'); if (!userGroupsOfTheUserSnapshot.exists()) { return null; diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index 685ffd39d..ce419e42d 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -681,7 +681,7 @@ async function fetchAoiFromHotTaskingManager(projectId: number | string): ( let response; try { response = await fetch( - `https://tasking-manager-tm4-production-api.hotosm.org/api/v2/projects/${projectId}/queries/aoi/?as_file=false`, + `https://tasking-manager-production-api.hotosm.org/api/v2/projects/${projectId}/queries/aoi/?as_file=false`, ); } catch { return { diff --git a/mapswipe_workers/python_scripts/extract_project_population_stats.py b/mapswipe_workers/python_scripts/extract_project_population_stats.py new file mode 100644 index 000000000..072f57f35 --- /dev/null +++ b/mapswipe_workers/python_scripts/extract_project_population_stats.py @@ -0,0 +1,143 @@ +import argparse +import os +import warnings + +import geopandas as gpd +import pandas as pd +import rasterio +import requests +from exactextract import exact_extract +from tqdm import tqdm + +warnings.filterwarnings("ignore") + + +def project_list(id_file): + """Reads Mapswipe project IDs from the user input file""" + + with open(id_file, "r") as file: + ids = file.read().strip() + + project_list = ids.split(",") + project_list = [id.strip() for id in project_list] + + return project_list + + +def population_raster_download(): + """Downloads 1km resolution global population raster for 2020 from WorldPop to the current working directory.""" + + url = "https://data.worldpop.org/GIS/Population/Global_2000_2020/2020/0_Mosaicked/ppp_2020_1km_Aggregated.tif" + + output_file = "ppp_2020_1km_Aggregated.tif" + + output_file_path = os.path.join(os.getcwd(), output_file) + + if os.path.exists(output_file_path): + + print("Population raster already exists. Moving to next steps......") + return output_file_path + + else: + + response = requests.get(url, stream=True) + size = int(response.headers.get("content-length", 0)) + block_size = 1024 + try: + with open(output_file, "wb") as file, tqdm( + desc="Downloading population raster", + total=size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(block_size): + if chunk: + file.write(chunk) + bar.update(len(chunk)) + + print("Download complete:", output_file_path) + return output_file_path + + except requests.RequestException as e: + print(f"Error downloading data: {e}") + + +def population_count(list, dir, raster): + """Gets boundary data for projects from Mapswipe API and calculates zonal statistics + with global population raster and individual project boundaries.""" + + dict = {} + worldpop = rasterio.open(raster) + + for id in list: + url = f"https://apps.mapswipe.org/api/project_geometries/project_geom_{id}.geojson" + response = requests.get(url) + + try: + geojson = response.json() + for feature in geojson["features"]: + geometry = feature.get("geometry", {}) + if "coordinates" in geometry: + if geometry["type"] == "Polygon": + geometry["coordinates"] = [ + [[coord[0], coord[1]] for coord in polygon] + for polygon in geometry["coordinates"] + ] + elif geometry["type"] == "MultiPolygon": + geometry["coordinates"] = [ + [ + [[coord[0], coord[1]] for coord in polygon] + for polygon in multipolygon + ] + for multipolygon in geometry["coordinates"] + ] + gdf = gpd.GeoDataFrame.from_features(geojson["features"]) + gdf.set_crs("EPSG:4326", inplace=True) + no_of_people = exact_extract(worldpop, gdf, "sum") + no_of_people = round(no_of_people[0]["properties"]["sum"]) + + dict[id] = no_of_people + + except requests.RequestException as e: + print(f"Error in retrieval of project boundary from Mapswipe: {e}") + + df = pd.DataFrame( + dict.items(), columns=["Project_IDs", "Number of people impacted"] + ) + + df["Project_IDs"] = "https://mapswipe.org/en/projects/" + df["Project_IDs"] + + df.to_csv(f"{dir}/projects_population.csv") + + print(f"CSV file successfully created at {dir}/number_of_people_impacted.csv") + + +if __name__ == "__main__": + """Generates population stats for individual Mapswipe projects""" + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", + "--text_file", + help=( + "Path to the text file containing project IDs from Mapswipe. The file should contain IDs in this manner: " + "-O8kulfxD4zRYQ2T1aXf, -O8kyOCreRGklW15n8RU, -O8kzSy9105axIPOAJjO, -OAwWv9rnJqPXTpWxO8-, " + "-OB-tettI2np7t3Gpu-k" + ), + type=str, + required=True, + ) + parser.add_argument( + "-o", + "--output_directory", + help="Path to the directory to store the output", + type=str, + required=True, + ) + args = parser.parse_args() + + population_count( + project_list(args.text_file), + args.output_directory, + population_raster_download(), + )