diff --git a/.github/linters/.jscpd.json b/.github/linters/.jscpd.json index 8665357..ebcc5ba 100644 --- a/.github/linters/.jscpd.json +++ b/.github/linters/.jscpd.json @@ -1,3 +1,3 @@ { - "threshold": 32 + "ignore": ["**/java/**","**/csharp/**","**/python/**"] } \ No newline at end of file diff --git a/.github/workflows/csharp-darwin-snippets.yaml b/.github/workflows/csharp-darwin-snippets.yaml new file mode 100644 index 0000000..6696aab --- /dev/null +++ b/.github/workflows/csharp-darwin-snippets.yaml @@ -0,0 +1,47 @@ +name: csharp darwin snippets + +on: [pull_request, workflow_dispatch] + +permissions: + contents: read + pull-requests: write + +jobs: + csharp-darwin-snippets: + runs-on: macos-latest + strategy: + matrix: + include: + - dotnet-version: "8" + + steps: + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.dotnet-version }} + + - name: install Senzing SDK + uses: senzing-factory/github-action-install-senzing-sdk@v2 + with: + senzingsdk-repository-path: ${{ secrets.SENZING_OSX_BETA_REPOSITORY }} + + - name: configure local nuget repo + run: | + find /opt/senzing/er/sdk/dotnet/ -name "Senzing.Sdk*" | xargs -0 -L 1 basename + sdk_package=$(find /opt/senzing/er/sdk/dotnet/ -name "Senzing.Sdk*" | xargs -0 -L 1 basename) + mkdir -p ~/dev/nuget/packages + dotnet nuget add source ~/dev/nuget/packages -n dev + dotnet nuget push /opt/senzing/er/sdk/dotnet/"${sdk_package}" --source dev + + - name: run csharp snippets + env: + DYLD_LIBRARY_PATH: "/opt/senzing/er/lib" + SENZING_DATA_DIR: "/opt/senzing/er/data" + run: | + cd "${GITHUB_WORKSPACE}"/csharp/runner + dotnet add SnippetRunner package Senzing.Sdk --version 4.0.0-beta + dotnet run --project SnippetRunner all diff --git a/.github/workflows/csharp-linux-snippets.yaml b/.github/workflows/csharp-linux-snippets.yaml new file mode 100644 index 0000000..0c5dfd1 --- /dev/null +++ b/.github/workflows/csharp-linux-snippets.yaml @@ -0,0 +1,51 @@ +name: csharp linux snippets + +on: + push: + branches-ignore: [main] + pull_request: + branches: [main] + +permissions: + contents: read + pull-requests: write + +jobs: + csharp-linux-snippets: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - dotnet-version: "8" + + steps: + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.dotnet-version }} + + - name: install Senzing runtime + uses: senzing-factory/github-action-install-senzing-sdk@v2 + with: + packages-to-install: "senzingsdk-runtime senzingsdk-setup" + senzingsdk-repository-package: ${{ secrets.SENZING_APT_BETA_STAGING_REPOSITORY_PACKAGE }} + senzingsdk-repository-path: ${{ secrets.SENZING_APT_BETA_STAGING_REPOSITORY_URL }} + + - name: configure local nuget repo + run: | + sdk_package=$(grep -Rl "Senzing.Sdk" /opt/senzing/er/sdk/dotnet/ | xargs -L 1 basename) + mkdir -p ~/dev/nuget/packages + dotnet nuget add source ~/dev/nuget/packages -n dev + dotnet nuget push /opt/senzing/er/sdk/dotnet/"${sdk_package}" --source dev + + - name: run csharp snippets + env: + LD_LIBRARY_PATH: "/opt/senzing/er/lib" + run: | + cd "${GITHUB_WORKSPACE}"/csharp/runner + dotnet add SnippetRunner package Senzing.Sdk --version 4.0.0-beta + dotnet run --project SnippetRunner all diff --git a/.github/workflows/csharp-windows-snippets.yaml b/.github/workflows/csharp-windows-snippets.yaml new file mode 100644 index 0000000..0a04f17 --- /dev/null +++ b/.github/workflows/csharp-windows-snippets.yaml @@ -0,0 +1,50 @@ +name: csharp windows snippets + +on: [pull_request, workflow_dispatch] + +permissions: + contents: read + pull-requests: write + +jobs: + csharp-windows-snippets: + runs-on: windows-latest + strategy: + matrix: + include: + - dotnet-version: "8" + + steps: + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.dotnet-version }} + + - name: install Senzing SDK + uses: senzing-factory/github-action-install-senzing-sdk@v2 + with: + senzingsdk-repository-path: ${{ secrets.SENZING_WIN_BETA_REPOSITORY }} + + - name: configure local nuget repo + run: | + $Env:sdk_package = Get-ChildItem -Path "C:\Program Files\Senzing\er\sdk\dotnet\" -Recurse | Where-Object {$_.Name -match "^Senzing.SDK*"} | Select-Object -ExpandProperty Name + mkdir $Env:USERPROFILE\dev\nuget\packages + cd $Env:USERPROFILE\dev\nuget\packages + dir + dotnet nuget add source $Env:USERPROFILE\dev\nuget\packages -n senzing + dotnet nuget push "C:\Program Files\Senzing\er\sdk\dotnet\${Env:sdk_package}" --source senzing + dotnet nuget list source + + - name: Add to "Path" environment variable + run: | + Add-Content $env:GITHUB_PATH "C:\Program Files\Senzing\er\lib" + + - name: run csharp snippets + run: | + echo $Env:Path + cd ${Env:GITHUB_WORKSPACE}/csharp/runner + dotnet run --project SnippetRunner all diff --git a/.github/workflows/java-darwin-snippets.yaml b/.github/workflows/java-darwin-snippets.yaml new file mode 100644 index 0000000..d946e8e --- /dev/null +++ b/.github/workflows/java-darwin-snippets.yaml @@ -0,0 +1,46 @@ +name: java darwin snippets + +on: [pull_request, workflow_dispatch] + +permissions: + contents: read + pull-requests: write + +jobs: + java-darwin-snippets: + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + java-version: ["17", "21"] + java-distribution: ["temurin"] + + steps: + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v4 + with: + distribution: ${{ matrix.java-distribution }} + java-version: ${{ matrix.java-version }} + + - name: install Senzing SDK + uses: senzing-factory/github-action-install-senzing-sdk@v2 + with: + senzingsdk-repository-path: ${{ secrets.SENZING_OSX_BETA_REPOSITORY }} + + - name: build with Maven + env: + SENZING_DIR: "/opt/senzing/er" + run: | + cd "${GITHUB_WORKSPACE}"/java + mvn clean install + + - name: run java snippets + env: + DYLD_LIBRARY_PATH: "/opt/senzing/er/lib" + run: | + cd "${GITHUB_WORKSPACE}"/java + java -jar target/sz-sdk-snippets.jar all diff --git a/.github/workflows/java-linux-snippets.yaml b/.github/workflows/java-linux-snippets.yaml new file mode 100644 index 0000000..07136da --- /dev/null +++ b/.github/workflows/java-linux-snippets.yaml @@ -0,0 +1,52 @@ +name: java linux snippets + +on: + push: + branches-ignore: [main] + pull_request: + branches: [main] + +permissions: + contents: read + pull-requests: write + +jobs: + java-linux-snippets: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java-version: ["17", "21"] + java-distribution: ["temurin"] + + steps: + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: ${{ matrix.java-distribution }} + + - name: install Senzing runtime + uses: senzing-factory/github-action-install-senzing-sdk@v2 + with: + packages-to-install: "senzingsdk-runtime senzingsdk-setup" + senzingsdk-repository-package: ${{ secrets.SENZING_APT_BETA_STAGING_REPOSITORY_PACKAGE }} + senzingsdk-repository-path: ${{ secrets.SENZING_APT_BETA_STAGING_REPOSITORY_URL }} + + - name: build with Maven + env: + SENZING_DIR: "/opt/senzing/er" + run: | + cd "${GITHUB_WORKSPACE}"/java + mvn clean package + + - name: run java snippets + env: + LD_LIBRARY_PATH: "/opt/senzing/er/lib:${LD_LIBRARY_PATH}" + run: | + cd "${GITHUB_WORKSPACE}"/java + java -jar target/sz-sdk-snippets.jar all diff --git a/.github/workflows/java-windows-snippets.yaml b/.github/workflows/java-windows-snippets.yaml new file mode 100644 index 0000000..cef76ab --- /dev/null +++ b/.github/workflows/java-windows-snippets.yaml @@ -0,0 +1,48 @@ +name: java windows snippets + +on: [pull_request, workflow_dispatch] + +permissions: + contents: read + pull-requests: write + +jobs: + java-windows-snippets: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + java-version: ["17", "21"] + java-distribution: ["temurin"] + + steps: + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: ${{ matrix.java-distribution }} + + - name: install Senzing SDK + uses: senzing-factory/github-action-install-senzing-sdk@v2 + with: + senzingsdk-repository-path: ${{ secrets.SENZING_WIN_BETA_REPOSITORY }} + + - name: build with Maven + env: + SENZING_DIR: 'C:\Program Files\Senzing\er' + run: | + cd "${Env:GITHUB_WORKSPACE}/java" + mvn clean install + + - name: Add to "Path" environment variable + run: | + Add-Content $env:GITHUB_PATH "C:\Program Files\Senzing\er\lib" + + - name: run java snippets + run: | + cd "${Env:GITHUB_WORKSPACE}/java" + java -jar target/sz-sdk-snippets.jar all diff --git a/csharp/runner/SnippetRunner/InstallLocations.cs b/csharp/runner/SnippetRunner/InstallLocations.cs index dd3bb60..9d57348 100644 --- a/csharp/runner/SnippetRunner/InstallLocations.cs +++ b/csharp/runner/SnippetRunner/InstallLocations.cs @@ -188,46 +188,22 @@ private static bool IsDirectory(string path) string defaultInstallPath; string? defaultConfigPath = null; - - // check if we are building within the dev structure - string[] directoryStructure = ["net8.0", "*", "bin", "Senzing.Sdk.Tests", "sz-sdk-csharp", "csharp", "g2", "apps", "dev"]; - DirectoryInfo? workingDir = new DirectoryInfo(Environment.CurrentDirectory); - DirectoryInfo? previousDir = null; - bool devStructure = true; - foreach (var dirName in directoryStructure) - { - if (workingDir == null) break; - if (!dirName.Equals("*", OrdinalIgnoreCase) && !workingDir.Name.Equals(dirName, OrdinalIgnoreCase)) - { - devStructure = false; - break; - } - previousDir = workingDir; - workingDir = workingDir.Parent; - } - DirectoryInfo? devDistDir = (devStructure && previousDir != null) - ? new DirectoryInfo(Path.Combine(previousDir.FullName, "dist")) : null; + string defaultSupportPath; switch (Environment.OSVersion.Platform) { case PlatformID.Win32NT: - defaultInstallPath = (devDistDir == null) - ? "C:\\Program Files\\Senzing\\er" : devDistDir.FullName; + defaultInstallPath = "C:\\Program Files\\Senzing\\er"; + defaultSupportPath = "C:\\Program Files\\Senzing\\er\\data"; break; case PlatformID.MacOSX: - defaultInstallPath = (devDistDir == null) - ? "/opt/senzing/er" : devDistDir.FullName; + defaultInstallPath = "/opt/senzing/er"; + defaultSupportPath = "/opt/senzing/er/data"; break; case PlatformID.Unix: - if (devDistDir == null) - { - defaultInstallPath = "/opt/senzing/er"; - defaultConfigPath = "/etc/opt/senzing"; - } - else - { - defaultInstallPath = devDistDir.FullName; - } + defaultInstallPath = "/opt/senzing/er"; + defaultConfigPath = "/etc/opt/senzing"; + defaultSupportPath = "/opt/senzing/data"; break; default: throw new NotSupportedException( @@ -314,104 +290,36 @@ private static bool IsDirectory(string path) return null; } + // check if an explicit support path has been specified if (supportPath == null || supportPath.Trim().Length == 0) { - // try to determine the support path - DirectoryInfo? installParent = installDir.Parent; - DirectoryInfo? dataRoot = (installParent != null) - ? new DirectoryInfo(Path.Combine(installParent.FullName, "data")) - : null; - - if (dataRoot != null && dataRoot.Exists && IsDirectory(dataRoot.FullName)) + // check if using a dev build + if ("dist".Equals(installDir.Name, OrdinalIgnoreCase)) { - DirectoryInfo? versionFile - = new DirectoryInfo( - Path.Combine(installDir.FullName, "szBuildVersion.json")); - - string? dataVersion = null; - if (versionFile != null && versionFile.Exists) - { - - String text = File.ReadAllText(versionFile.FullName, UTF8); - JsonDocument jsonDoc = JsonDocument.Parse(text); - JsonElement rootElem = jsonDoc.RootElement; - JsonElement dataVerElem = rootElem.GetProperty("DATA_VERSION"); - dataVersion = dataVerElem.GetString(); - } - - // try the data version directory - supportDir = (dataVersion == null) - ? null - : new DirectoryInfo( - Path.Combine(dataRoot.FullName, dataVersion.Trim())); - - // check if data version was not found - if (supportDir == null || !supportDir.Exists) + // use the "data" sub-directory of the dev build + supportDir = new DirectoryInfo( + Path.Combine(installDir.FullName, "data")); + } + else + { + switch (Environment.OSVersion.Platform) { - Regex regex = new Regex("\\d+\\.\\d+\\.\\d+"); - // look to see if we only have one data version installed - string[] dirs = Directory.GetDirectories(dataRoot.FullName); - List versionDirs - = new List(); - foreach (string dir in dirs) - { - DirectoryInfo versionDir = new DirectoryInfo(dir); - if (!regex.IsMatch(versionDir.Name)) continue; - versionDirs.Add(versionDir); - } - if (versionDirs.Count == 1 && supportDir == null) - { - // use the single data version found - supportDir = versionDirs[0]; - } - else if (versionDirs.Count > 1) - { - Console.Error.WriteLine( - "Could not infer support directory. Multiple data " - + "directory versions at: "); - Console.Error.WriteLine(" " + dataRoot); - if (supportDir != null) - { - Console.Error.WriteLine(); - Console.Error.WriteLine("Expected to find: " + supportDir); - } - throw new InvalidOperationException( - ((supportDir == null) - ? "Could not infer support directory." - : "Could not find support directory (" + supportDir + ").") - + " Multiple data directory versions found at: " - + dataRoot); - } - else - { - // no version directories were found, maybe the data root is - // the actual support directory (mapped in a docker image) - string[] files = Directory.GetFiles(dataRoot.FullName); - List ibmFiles = new List(); - foreach (string file in files) - { - if (file.EndsWith(".ibm", OrdinalIgnoreCase)) - { - ibmFiles.Add(new FileInfo(file)); - } - } - DirectoryInfo? libPostalDir = new DirectoryInfo( - Path.Combine(dataRoot.FullName, "libpostal")); - - // require the .ibm files and libpostal to exist - if (ibmFiles.Count > 0 && libPostalDir.Exists) - { - supportDir = dataRoot; - } - } + case PlatformID.Win32NT: + defaultSupportPath = Path.Combine(installDir.FullName, "data"); + break; + case PlatformID.MacOSX: + defaultSupportPath = Path.Combine(installDir.FullName, "data"); + break; + case PlatformID.Unix: + break; + default: + throw new NotSupportedException( + "Unsupported Operating System: " + + Environment.OSVersion.Platform); } - } - if (supportDir == null) - { - // use the default path - supportDir = new DirectoryInfo( - Path.Combine(installDir.FullName, "data")); + // no explicit path, try the default support path + supportDir = new DirectoryInfo(defaultSupportPath); } } diff --git a/csharp/runner/SnippetRunner/Program.cs b/csharp/runner/SnippetRunner/Program.cs index ebd212f..6193f9b 100644 --- a/csharp/runner/SnippetRunner/Program.cs +++ b/csharp/runner/SnippetRunner/Program.cs @@ -130,7 +130,8 @@ SortedDictionary> snippetsMap try { settingsJson = JsonNode.Parse(settings)?.AsObject(); - if (settingsJson == null) { + if (settingsJson == null) + { throw new Exception("Setting must be a JSON object: " + settings); } } @@ -537,21 +538,27 @@ static void ExecuteSnippet(string snippet, bool exited = process.WaitForExit(delay); if (!exited && !process.HasExited) { - expectedExitValue = SigtermExitCode; + expectedExitValue = (Environment.OSVersion.Platform == PlatformID.Win32NT) + ? 1 : SigtermExitCode; Console.WriteLine(); Console.WriteLine("Runner destroying " + snippet + " process..."); ProcessStartInfo killStartInfo - = new ProcessStartInfo("kill", "" + process.Id); + = (Environment.OSVersion.Platform == PlatformID.Win32NT) + ? new ProcessStartInfo("taskkill", ["/F", "/PID", "" + process.Id]) + : new ProcessStartInfo("kill", "" + process.Id); startInfo.WindowStyle = ProcessWindowStyle.Hidden; startInfo.UseShellExecute = false; Process? killer = Process.Start(killStartInfo); - if (killer == null) { + if (killer == null) + { process.Kill(true); - process.WaitForExit(); - } else { + process.WaitForExit(); + } + else + { killer.WaitForExit(); process.WaitForExit(); } @@ -629,10 +636,11 @@ static string SetupTempRepository(InstallLocations senzingInstall) } } - string supportPath = supportDir.FullName; - string configPath = configDir.FullName; - string resourcePath = resourcesDir.FullName; - string baseConfig = File.ReadAllText(configFile); + string supportPath = supportDir.FullName.Replace("\\", "\\\\"); + string configPath = configDir.FullName.Replace("\\", "\\\\"); ; + string resourcePath = resourcesDir.FullName.Replace("\\", "\\\\"); ; + string baseConfig = File.ReadAllText(configFile).Replace("\\", "\\\\"); + string databasePath = databaseFile.Replace("\\", "\\\\"); string settings = $$""" { "PIPELINE": { @@ -641,7 +649,7 @@ static string SetupTempRepository(InstallLocations senzingInstall) "RESOURCEPATH": "{{resourcePath}}" }, "SQL": { - "CONNECTION": "sqlite3://na:na@{{databaseFile}}" + "CONNECTION": "sqlite3://na:na@{{databasePath}}" } } """.Trim(); @@ -655,6 +663,11 @@ static string SetupTempRepository(InstallLocations senzingInstall) configMgr.SetDefaultConfigID(configID); } + catch (Exception) + { + Console.Error.WriteLine(settings); + throw; + } finally { env.Destroy(); diff --git a/csharp/runner/SnippetRunner/Resources/redo/RedoContinuous.properties b/csharp/runner/SnippetRunner/Resources/redo/RedoContinuous.properties index 16be70f..280d48b 100644 --- a/csharp/runner/SnippetRunner/Resources/redo/RedoContinuous.properties +++ b/csharp/runner/SnippetRunner/Resources/redo/RedoContinuous.properties @@ -4,4 +4,4 @@ source.2=REFERENCE load.0=/data/truthset/customers.jsonl load.1=/data/truthset/reference.jsonl load.2=/data/truthset/watchlist.jsonl -destroyAfter=15000 +destroyAfter=30000 diff --git a/csharp/runner/SnippetRunner/Resources/redo/RedoContinuousViaFutures.properties b/csharp/runner/SnippetRunner/Resources/redo/RedoContinuousViaFutures.properties index 16be70f..280d48b 100644 --- a/csharp/runner/SnippetRunner/Resources/redo/RedoContinuousViaFutures.properties +++ b/csharp/runner/SnippetRunner/Resources/redo/RedoContinuousViaFutures.properties @@ -4,4 +4,4 @@ source.2=REFERENCE load.0=/data/truthset/customers.jsonl load.1=/data/truthset/reference.jsonl load.2=/data/truthset/watchlist.jsonl -destroyAfter=15000 +destroyAfter=30000 diff --git a/csharp/runner/SnippetRunner/Resources/redo/RedoWithInfoContinuous.properties b/csharp/runner/SnippetRunner/Resources/redo/RedoWithInfoContinuous.properties index 16be70f..280d48b 100644 --- a/csharp/runner/SnippetRunner/Resources/redo/RedoWithInfoContinuous.properties +++ b/csharp/runner/SnippetRunner/Resources/redo/RedoWithInfoContinuous.properties @@ -4,4 +4,4 @@ source.2=REFERENCE load.0=/data/truthset/customers.jsonl load.1=/data/truthset/reference.jsonl load.2=/data/truthset/watchlist.jsonl -destroyAfter=15000 +destroyAfter=30000 diff --git a/csharp/snippets/deleting/DeleteViaFutures/Program.cs b/csharp/snippets/deleting/DeleteViaFutures/Program.cs index 231c69e..0c228c4 100644 --- a/csharp/snippets/deleting/DeleteViaFutures/Program.cs +++ b/csharp/snippets/deleting/DeleteViaFutures/Program.cs @@ -159,6 +159,8 @@ TaskScheduler taskScheduler } finally { + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/csharp/snippets/deleting/DeleteViaLoop/Program.cs b/csharp/snippets/deleting/DeleteViaLoop/Program.cs index 18ca6cc..f49596a 100644 --- a/csharp/snippets/deleting/DeleteViaLoop/Program.cs +++ b/csharp/snippets/deleting/DeleteViaLoop/Program.cs @@ -130,6 +130,8 @@ } finally { + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/csharp/snippets/deleting/DeleteWithInfoViaFutures/Program.cs b/csharp/snippets/deleting/DeleteWithInfoViaFutures/Program.cs index 094656c..a930e2a 100644 --- a/csharp/snippets/deleting/DeleteWithInfoViaFutures/Program.cs +++ b/csharp/snippets/deleting/DeleteWithInfoViaFutures/Program.cs @@ -160,6 +160,8 @@ TaskScheduler taskScheduler } finally { + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/csharp/snippets/loading/LoadViaFutures/Program.cs b/csharp/snippets/loading/LoadViaFutures/Program.cs index 8855cb6..9821743 100644 --- a/csharp/snippets/loading/LoadViaFutures/Program.cs +++ b/csharp/snippets/loading/LoadViaFutures/Program.cs @@ -49,7 +49,7 @@ TaskScheduler taskScheduler // keep track of the pending tasks and don't backlog too many for memory's sake IList<(Task, Record)> pendingFutures = new List<(Task, Record)>(MaximumBacklog); -FileStream fs = new FileStream(filePath, FileMode.Open); +FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); try { // create a reader @@ -159,6 +159,9 @@ TaskScheduler taskScheduler } finally { + // close the file stream + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/csharp/snippets/loading/LoadViaLoop/Program.cs b/csharp/snippets/loading/LoadViaLoop/Program.cs index 36c933d..fe61630 100644 --- a/csharp/snippets/loading/LoadViaLoop/Program.cs +++ b/csharp/snippets/loading/LoadViaLoop/Program.cs @@ -31,7 +31,7 @@ string filePath = (args.Length > 0) ? args[0] : DefaultFilePath; -FileStream fs = new FileStream(filePath, FileMode.Open); +FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); try { // create a reader @@ -130,6 +130,8 @@ } finally { + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/csharp/snippets/loading/LoadViaQueue/Program.cs b/csharp/snippets/loading/LoadViaQueue/Program.cs index 991edb0..bb3c844 100644 --- a/csharp/snippets/loading/LoadViaQueue/Program.cs +++ b/csharp/snippets/loading/LoadViaQueue/Program.cs @@ -32,10 +32,9 @@ string filePath = (args.Length > 0) ? args[0] : DefaultFilePath; -FileStream fs = new FileStream(filePath, FileMode.Open); Thread producer = new Thread(() => { - FileStream fs = new FileStream(filePath, FileMode.Open); + FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); try { StreamReader rdr = new StreamReader(fs, Encoding.UTF8); @@ -71,6 +70,7 @@ } finally { + fs.Close(); recordQueue.CompleteAdding(); } }); diff --git a/csharp/snippets/loading/LoadWithInfoViaFutures/Program.cs b/csharp/snippets/loading/LoadWithInfoViaFutures/Program.cs index 7c586d0..75f2097 100644 --- a/csharp/snippets/loading/LoadWithInfoViaFutures/Program.cs +++ b/csharp/snippets/loading/LoadWithInfoViaFutures/Program.cs @@ -50,7 +50,7 @@ TaskScheduler taskScheduler IList<(Task, Record)> pendingFutures = new List<(Task, Record)>(MaximumBacklog); -FileStream fs = new FileStream(filePath, FileMode.Open); +FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); try { // create a reader @@ -161,6 +161,8 @@ TaskScheduler taskScheduler } finally { + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/csharp/snippets/loading/LoadWithStatsViaLoop/Program.cs b/csharp/snippets/loading/LoadWithStatsViaLoop/Program.cs index 64839c0..31a672b 100644 --- a/csharp/snippets/loading/LoadWithStatsViaLoop/Program.cs +++ b/csharp/snippets/loading/LoadWithStatsViaLoop/Program.cs @@ -31,7 +31,7 @@ string filePath = (args.Length > 0) ? args[0] : DefaultFilePath; -FileStream fs = new FileStream(filePath, FileMode.Open); +FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); try { // create a reader @@ -151,6 +151,8 @@ } finally { + fs.Close(); + // IMPORTANT: make sure to destroy the environment env.Destroy(); diff --git a/java/README.md b/java/README.md index d5efc1a..f47b23e 100644 --- a/java/README.md +++ b/java/README.md @@ -6,6 +6,26 @@ The Java snippets are contained in the `snippets` directory under various Java p The Java snippets can built using the `pom.xml` in this directory using `mvn package`. The result will be the `sz-sdk-snippets.jar` file in the `target` directory. +1. First, set the `SENZING_DIR` environment variable so the `pom.xml` can locate +the `sz-sdk.jar`. + - Linux/macOS: + + ```console + export SENZING_DIR=/opt/senzing/er + ``` + + - Windows: + + ```console + set SENZING_DIR=C:\Program Files\Senzing\er + ``` + +1. Run the maven build: + + ```console + mvn package + ``` + ## Running There are several ways to run the code snippets. diff --git a/java/pom.xml b/java/pom.xml index ab13857..ada36f7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ sz-sdk 4.0.0-beta.2.0 system - ${SENZING_DIR}/lib/sz-sdk.jar + ${SENZING_DIR}/sdk/java/sz-sdk.jar org.glassfish @@ -88,7 +88,7 @@ false - ${SENZING_DIR}/lib/sz-sdk.jar + ${SENZING_DIR}/sdk/java/sz-sdk.jar diff --git a/java/runner/java/com/senzing/runner/InstallLocations.java b/java/runner/java/com/senzing/runner/InstallLocations.java index 63fcd51..efd0cc4 100644 --- a/java/runner/java/com/senzing/runner/InstallLocations.java +++ b/java/runner/java/com/senzing/runner/InstallLocations.java @@ -1,14 +1,11 @@ package com.senzing.runner; -import javax.json.JsonObject; import java.io.File; import java.io.StringWriter; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; -import static com.senzing.runner.Utilities.*; - /** * Describes the directories on disk used to find the Senzing product * installation and the support directories. @@ -165,14 +162,18 @@ public static InstallLocations findLocations() { try { String defaultInstallPath; String defaultConfigPath = null; + String defaultSupportPath = null; if (windows) { - defaultInstallPath = "C:\\\\Program Files\\Senzing\\er"; + defaultInstallPath = "C:\\Program Files\\Senzing\\er"; + defaultSupportPath = "C:\\Program Files\\Senzing\\er\\data"; } else if (macOS) { defaultInstallPath = "/opt/senzing/er"; + defaultSupportPath = "/opt/senzing/er/data"; } else { defaultInstallPath = "/opt/senzing/er"; - defaultConfigPath = "/etc/opt/senzing"; + defaultConfigPath = "/etc/opt/senzing"; + defaultSupportPath = "/opt/senzing/data"; } // set the install path if one has been provided @@ -249,65 +250,17 @@ public static InstallLocations findLocations() { return null; } + // check if an explicit support path has been specified if (supportPath == null || supportPath.trim().length() == 0) { - // try to determine the support path - File installParent = installDir.getParentFile(); - File dataRoot = new File(installParent, "data"); - if (dataRoot.exists() && dataRoot.isDirectory()) { - File versionFile = new File(installDir, "szBuildVersion.json"); - String dataVersion = null; - if (versionFile.exists()) { - String text = readTextFileAsString(versionFile, UTF_8); - JsonObject jsonObject = parseJsonObject(text); - dataVersion = (jsonObject.containsKey("DATA_VERSION") - ? jsonObject.getString("DATA_VERSION") : null); - } - - // try the data version directory - supportDir = (dataVersion == null) ? null : new File(dataRoot, dataVersion.trim()); - - // check if data version was not found - if (supportDir == null || !supportDir.exists()) { - // look to see if we only have one data version installed - File[] versionDirs = dataRoot.listFiles(f -> { - return f.getName().matches("\\d+\\.\\d+\\.\\d+"); - }); - if (versionDirs.length == 1 && supportDir == null) { - // use the single data version found - supportDir = versionDirs[0]; - - } else if (versionDirs.length > 1) { - System.err.println( - "Could not infer support directory. Multiple data " - + "directory versions at: "); - System.err.println(" " + dataRoot); - if (supportDir != null) { - System.err.println(); - System.err.println("Expected to find: " + supportDir); - } - throw new IllegalStateException( - ((supportDir == null) ? "Could not infer support directory." - : "Could not find support directory (" + supportDir + ").") - + " Multiple data directory versions found at: " + dataRoot); - } else { - // no version directories were found, maybe the data root is - // the actual support directory (mapped in a docker image) - File[] ibmFiles = dataRoot.listFiles(f -> { - return f.getName().toLowerCase().endsWith(".ibm"); - }); - File libPostalDir = new File(dataRoot, "libpostal"); - - // require the .ibm files and libpostal to exist - if (ibmFiles.length > 0 && libPostalDir.exists()) { - supportDir = dataRoot; - } - } - } - - } - if (supportDir == null) { - // use the default path + // check if using a dev build + if ("dist".equals(installDir.getName())) { + // use the "data" sub-directory of the dev build supportDir = new File(installDir, "data"); + } else if (windows || macOS) { + supportDir = new File(installDir, "data"); + } else { + // no explicit path, try the default support path + supportDir = new File(defaultSupportPath); } } else { diff --git a/java/runner/java/com/senzing/runner/SnippetRunner.java b/java/runner/java/com/senzing/runner/SnippetRunner.java index 92a42b9..d0689ae 100644 --- a/java/runner/java/com/senzing/runner/SnippetRunner.java +++ b/java/runner/java/com/senzing/runner/SnippetRunner.java @@ -35,6 +35,26 @@ public class SnippetRunner { private static final int SIGTERM_EXIT_CODE = 143; + private static final boolean WINDOWS; + private static final boolean MACOS; + + static { + final String osName = System.getProperty("os.name"); + + boolean windows = false; + boolean macOS = false; + + String lowerOSName = osName.toLowerCase().trim(); + if (lowerOSName.startsWith("windows")) { + windows = true; + } else if (lowerOSName.startsWith("mac") || lowerOSName.indexOf("darwin") >= 0) { + macOS = true; + } + + WINDOWS = windows; + MACOS = macOS; + } + /** * Harness for running one or more of the code snippets. * @@ -316,12 +336,13 @@ private static void executeSnippet(String snippet, long delay = Long.parseLong(propValue); boolean exited = process.waitFor(delay, TimeUnit.MILLISECONDS); if (!exited && process.isAlive()) { - expectedExitValue = SIGTERM_EXIT_CODE; + expectedExitValue = (WINDOWS) ? 1 : SIGTERM_EXIT_CODE; System.out.println(); System.out.println("Runner destroying " + snippet + " process..."); // NOTE: using process.destroy() does not trigger the registered // shutdown hooks in the snippet sub-process for some reason - Process killer = runtime.exec("kill " + process.pid()); + Process killer = runtime.exec( + ((WINDOWS) ? "taskkill /F /PID " : "kill ") + process.pid()); killer.waitFor(); // wait for the kill process to complete } exitValue = process.waitFor(); @@ -373,27 +394,15 @@ private static void printUsage(SortedMap> snippetMap) private static String getJarPath() throws RuntimeException { try { - final String osName = System.getProperty("os.name"); - - boolean windows = false; - boolean macOS = false; - - String lowerOSName = osName.toLowerCase().trim(); - if (lowerOSName.startsWith("windows")) { - windows = true; - } else if (lowerOSName.startsWith("mac") || lowerOSName.indexOf("darwin") >= 0) { - macOS = true; - } - String resourceName = SnippetRunner.class.getSimpleName() + ".class"; String url = SnippetRunner.class.getResource(resourceName).toString(); String jarPath = url.replaceAll("jar:file:(.*\\.jar)\\!/.*\\.class", "$1"); - if (windows && jarPath.startsWith("/")) { + if (WINDOWS && jarPath.startsWith("/")) { jarPath = jarPath.replaceAll("[/]+([^/].*)", "$1"); } - if (windows && jarPath.startsWith("/")) { + if (WINDOWS && jarPath.startsWith("/")) { jarPath = jarPath.substring(1); } return jarPath; @@ -472,10 +481,10 @@ private static String setupTempRepository(InstallLocations senzingInstall) throw } } - String supportPath = senzingInstall.getSupportDirectory().getCanonicalPath(); - String configPath = configDir.getCanonicalPath(); - String resourcePath = resourcesDir.toString(); - String databasePath = databaseFile.getCanonicalPath(); + String supportPath = senzingInstall.getSupportDirectory().getCanonicalPath().replace("\\", "\\\\"); + String configPath = configDir.getCanonicalPath().replace("\\", "\\\\"); + String resourcePath = resourcesDir.toString().replace("\\", "\\\\"); + String databasePath = databaseFile.getCanonicalPath().replace("\\", "\\\\"); String baseConfig = readTextFileAsString(configFile, UTF_8); String settings = """ { @@ -497,6 +506,10 @@ private static String setupTempRepository(InstallLocations senzingInstall) throw long configId = configMgr.addConfig(baseConfig, "Default Config"); configMgr.setDefaultConfigId(configId); + } catch (SzException e) { + System.err.println(settings); + throw e; + } finally { env.destroy(); } diff --git a/java/runner/resources/redo/RedoContinuous.properties b/java/runner/resources/redo/RedoContinuous.properties index 16be70f..280d48b 100644 --- a/java/runner/resources/redo/RedoContinuous.properties +++ b/java/runner/resources/redo/RedoContinuous.properties @@ -4,4 +4,4 @@ source.2=REFERENCE load.0=/data/truthset/customers.jsonl load.1=/data/truthset/reference.jsonl load.2=/data/truthset/watchlist.jsonl -destroyAfter=15000 +destroyAfter=30000 diff --git a/java/runner/resources/redo/RedoContinuousViaFutures.properties b/java/runner/resources/redo/RedoContinuousViaFutures.properties index 16be70f..280d48b 100644 --- a/java/runner/resources/redo/RedoContinuousViaFutures.properties +++ b/java/runner/resources/redo/RedoContinuousViaFutures.properties @@ -4,4 +4,4 @@ source.2=REFERENCE load.0=/data/truthset/customers.jsonl load.1=/data/truthset/reference.jsonl load.2=/data/truthset/watchlist.jsonl -destroyAfter=15000 +destroyAfter=30000 diff --git a/java/runner/resources/redo/RedoWithInfoContinuous.properties b/java/runner/resources/redo/RedoWithInfoContinuous.properties index 16be70f..280d48b 100644 --- a/java/runner/resources/redo/RedoWithInfoContinuous.properties +++ b/java/runner/resources/redo/RedoWithInfoContinuous.properties @@ -4,4 +4,4 @@ source.2=REFERENCE load.0=/data/truthset/customers.jsonl load.1=/data/truthset/reference.jsonl load.2=/data/truthset/watchlist.jsonl -destroyAfter=15000 +destroyAfter=30000