Merge branch 'dev' into speed-shapefile-matching

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,73 @@
+# This file was created by CodeQL for this repository. The only change was 
+# removing 'python' and 'javascript' from the language array.
+#
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ dev ]
+  schedule:
+    - cron: '24 12 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/gradle.yml

@@ -33,14 +33,11 @@ jobs:
         fetch-depth: 0
     # Java setup step completes very fast, no need to run in a preconfigured docker container.
     - name: Set up JDK 11
-      uses: actions/setup-java@v1
+      uses: actions/setup-java@v3
       with:
         java-version: 11
-    - uses: actions/cache@v1
-      id: cache
-      with:
-        path: ~/.gradle/caches
-        key: gradle-caches
+        distribution: temurin
+        cache: 'gradle'
     - name: Show version string
       run: gradle -q printVersion | head -n1
     - name: Build and Test

README.md

@@ -7,7 +7,9 @@ We refer to the routing method as "realistic" because it works by planning door-
 
 We say "Real-world and Reimagined" networks because R5's networks are built from widely available open OSM and GTFS data describing baseline transportation systems, but R5 includes a system for applying light-weight patches to those networks for immediate, interactive scenario comparison.
 
-**Please note** that the Conveyal team does not provide technical support for third-party deployments of its analysis platform. We provide paid subscriptions to a cloud-based deployment of this system, which performs these complex calculations hundreds of times faster using a compute cluster. This project is open source primarily to ensure transparency and reproducibility in public planning and decision making processes, and in hopes that it may help researchers, students, and potential collaborators to understand and build upon our methodology.
+**Please note** that the Conveyal team does not provide technical support for third-party deployments. R5 is a component of a specialized commercial system, and we align development efforts with our roadmap and the needs of subscribers to our hosted service. This service is designed to facilitate secure online collaboration, user-friendly data management and scenario editing through a web interface, and complex calculations performed hundreds of times faster using a compute cluster. These design goals may not align well with other use cases. This project is open source primarily to ensure transparency and reproducibility in public planning and decision making processes, and in hopes that it may help researchers, students, and potential collaborators to understand and build upon our methodology.
+
+While the Conveyal team provides ongoing support and compatibility to subscribers, third-party projects using R5 as a library may not work with future releases. R5 does not currently expose a stable programming interface ("API" or "SDK"). As we release new features, previous functions and data types may change. The practical effect is that third-party wrappers or language bindings (e.g., for R or Python) may need to continue using an older release of R5 for feature compatibility (though not necessarily result compatibility, as the methods used in R5 are now relatively mature). 
 
 ## Methodology
 
@@ -19,7 +21,7 @@ For details on the core methods implemented in Conveyal Analysis and R5, see:
 
 ### Citations
 
-The Conveyal team is always eager to see cutting-edge uses of our software, so feel free to send us a copy of any thesis, report, or paper produced using this software. We also ask that any academic publications using this software cite the papers above, where relevant and appropriate.
+The Conveyal team is always eager to see cutting-edge uses of our software, so feel free to send us a copy of any thesis, report, or paper produced using this software. We also ask that any academic or research publications using this software cite the papers above, where relevant and appropriate.
 
 ## Configuration
 
@@ -52,7 +54,7 @@ By default, IntelliJ will follow common Gradle practice and build R5 using the "
 
 ## Structured Commit Messages
 
-We use structured commit messages to help generate changelogs and determine version numbers.
+We use structured commit messages to help generate changelogs.
 
 The first line of these messages is in the following format: `<type>(<scope>): <summary>` 
 
@@ -68,10 +70,6 @@ The `(<scope>)` is optional and is often a class name. The `<summary>` should be
 - devops: Changes to code that only affect deployment, logging, etc. No changes to user code.
 - chore: Any other changes causing no changes to user code.
 
-The body of the commit message (if any) should begin after one blank line. If the commit meets the definition of a major version change according to semantic versioning (e.g. a change in API visible to an external module), the commit message body should begin with `BREAKING CHANGE: <description>`.
-
-Presence of a `fix` commit in a release should increment the number in the third (PATCH) position.
-Presence of a `feat` commit in a release should increment the number in the second (MINOR) position.
-Presence of a `BREAKING CHANGE` commit in a release should increment the number in the first (MAJOR) position.
+The body of the commit message (if any) should begin after one blank line. 
 
-This is based on https://www.conventionalcommits.org.
+From 2018 to 2020, we used major/minor/patch release numbering as suggested by https://www.conventionalcommits.org. Starting in 2021, we switched to major/minor release numbering, incrementing the minor version with regular feature releases and the major version only when there are substantial changes to the cluster computing components of our system. Because there is no public API at this time, the conventional definition of breaking changes under semantic versioning does not apply. 

build.gradle

@@ -1,8 +1,8 @@
 plugins {
     id 'java'
-    id 'com.github.johnrengelman.shadow' version '6.0.0'
+    id 'com.github.johnrengelman.shadow' version '8.1.0'
     id 'maven-publish'
-    id 'com.palantir.git-version' version '0.12.3'
+    id 'com.palantir.git-version' version '2.0.0'
 }
 
 group = 'com.conveyal'
@@ -72,17 +72,19 @@ tasks.withType(JavaCompile) {
     options.encoding = 'UTF-8'
 }
 
-// A task to copy all dependencies of the project into a single directory
+// A task to copy all dependency JARs needed at runtime into a single directory
 task copyDependencies(type: Copy) {
-    from configurations.default
+    from(sourceSets.main.runtimeClasspath) {
+        include '*.jar'
+    }
     into 'dependencies'
 }
 
 // Run R5 as a local analysis backend with all dependencies on the classpath, without building a shadowJar.
 task runBackend (type: JavaExec) {
    dependsOn(build)
    classpath(sourceSets.main.runtimeClasspath)
-   main("com.conveyal.analysis.BackendMain")
+   mainClass = 'com.conveyal.analysis.BackendMain'
 }
 
 // Start up an analysis local backend from a shaded JAR and ask it to shut down immediately.
@@ -91,7 +93,7 @@ task runBackend (type: JavaExec) {
 task testShadowJarRunnable(type: JavaExec) {
     dependsOn(shadowJar)
     classpath(shadowJar.archiveFile.get())
-    main("com.conveyal.analysis.BackendMain")
+    mainClass = 'com.conveyal.analysis.BackendMain'
     jvmArgs("-Dconveyal.immediate.shutdown=true")
 }
 
@@ -123,12 +125,8 @@ repositories {
     // Put Open Source Geospatial before Maven Central to get JAI core, see https://stackoverflow.com/a/26993223
     maven { url 'https://repo.osgeo.org/repository/release/' }
     mavenCentral()
-    // TODO review whether we really need these repositories
+    // Polyline encoder 0.2 is now in Maven repo
     maven { url 'https://maven.conveyal.com' }
-    // Used for importing java projects from github (why do we need this?)
-    maven { url 'https://jitpack.io' }
-    // For the polyline encoder
-    maven { url 'https://nexus.axiomalaska.com/nexus/content/repositories/public-releases' }
 }
 
 // Exclude all JUnit 4 transitive dependencies - IntelliJ bug causes it to think we're using Junit 4 instead of 5.

src/main/java/com/conveyal/analysis/components/TaskScheduler.java

@@ -168,11 +168,39 @@ public final void run () {
             List<ApiTask> apiTaskList = tasks.stream()
                     .map(Task::toApiTask)
                     .collect(Collectors.toUnmodifiableList());
-            tasks.removeIf(t -> t.durationComplete().getSeconds() > 60);
+            // Purge old tasks.
+            tasks.removeIf(t -> t.durationComplete().toDays() > 3);
             return apiTaskList;
         }
     }
 
+    /**
+     * Return a single task. Does not purge old tasks.
+     */
+    public Task getTaskForUser (String userEmail, String taskId) {
+        synchronized (tasksForUser) {
+            Set<Task> tasks = tasksForUser.get(userEmail);
+            if (tasks == null) return null;
+            for (Task t : tasks) {
+                if (t.id.toString().equals(taskId)) {
+                    return t;
+                }
+            }
+            return null;
+        }
+    }
+
+    /**
+     * Remove a task. Returns false if task does not exist. Returns true if task exists and is properly removed.
+     */
+    public boolean removeTaskForUser (String userEmail, String taskId) {
+        synchronized (tasksForUser) {
+            Set<Task> tasks = tasksForUser.get(userEmail);
+            if (tasks == null) return false;
+            return tasks.removeIf(t -> t.id.toString().equals(taskId));
+        }
+    }
+
     // Q: Should the caller ever create its own Tasks, or if are tasks created here inside the TaskScheduler from
     // other raw information? Having the caller creating a Task seems like a good way to configure execution details
     // like heavy/light/periodic, and submit user information without passing it in. That task request could be separate