Merge branch 'master' into Issue160_import_lexer_grammars

Conflicts:
	contributors.txt
This commit is contained in:
Nicolas 2017-11-28 23:15:02 +13:00
commit 6239a640ce
9 changed files with 111 additions and 103 deletions

View File

@ -3,9 +3,9 @@
set -euo pipefail
cache_dir="$HOME/Library/Caches/Antlr4"
dotnet_url='https://download.microsoft.com/download/B/9/F/B9F1AF57-C14A-4670-9973-CDF47209B5BF/dotnet-dev-osx-x64.1.0.4.pkg'
dotnet_url='https://download.microsoft.com/download/F/4/F/F4FCB6EC-5F05-4DF8-822C-FF013DF1B17F/dotnet-dev-osx-x64.1.1.4.pkg'
dotnet_file=$(basename "$dotnet_url")
dotnet_shasum='63b5d99028cd8b2454736076106c96ba7d05f0fc'
dotnet_shasum='dc46d93716db8bea8cc3c668088cc9e39384b5a4'
thisdir=$(dirname "$0")

View File

@ -2,10 +2,16 @@ version: '4.7.1-SNAPSHOT+AppVeyor.{build}'
cache:
- '%USERPROFILE%\.m2'
- '%USERPROFILE%\.nuget\packages -> **\project.json'
image: Visual Studio 2017
build: off
build_script:
- mvn -DskipTests install --batch-mode
- msbuild runtime/CSharp/runtime/CSharp/Antlr4.vs2013.sln /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed
- msbuild /target:restore /target:rebuild /property:Configuration=Release /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
- msbuild ./runtime-testsuite/target/classes/CSharp/runtime/CSharp/Antlr4.vs2013.sln /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed
after_build:
- msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
test_script:
- mvn install -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode
artifacts:
- path: 'runtime\**\*.nupkg'
name: NuGet

View File

@ -165,10 +165,12 @@ YYYY/MM/DD, github id, Full name, email
2017/08/29, Eddy Reyes, eddy@mindsight.io
2017/09/09, brauliobz, Bráulio Bezerra, brauliobezerra@gmail.com
2017/09/11, sachinjain024, Sachin Jain, sachinjain024@gmail.com
2017/09/25, kaedvann, Rostislav Listerenko, r.listerenko@gmail.com
2017/10/06, bramp, Andrew Brampton, brampton@gmail.com
2017/10/15, simkimsia, Sim Kim Sia, kimcity@gmail.com
2017/10/27, Griffon26, Maurice van der Pot, griffon26@kfk4ever.com
2017/05/29, rlfnb, Ralf Neeb, rlfnb@rlfnb.de
2017/10/29, gendalph, Максим Прохоренко, Maxim\dotProhorenko@gm@il.com
2017/11/02, jasonmoo, Jason Mooberry, jason.mooberry@gmail.com
2017/11/24, zqlu.cn, Zhiqiang Lu, zqlu.cn@gmail.com
2017/11/28, niccroad, Nicolas Croad, nic.croad@gmail.com

View File

@ -32,8 +32,7 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t
* runtime/Python3/setup.py
* runtime/Python3/src/antlr4/Recognizer.py
* runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs
* runtime/CSharp/build/version.ps1
* runtime/CSharp/runtime/CSharp/Package.nuspec
* runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
* runtime/JavaScript/src/antlr4/package.json
* runtime/JavaScript/src/antlr4/Recognizer.js
* runtime/Cpp/VERSION
@ -245,75 +244,55 @@ popd
### CSharp
*Publishing to Nuget from Linux/MacOSX*
*Publishing to Nuget from Windows*
**Install the pre-requisites**
Of course you need Mono and `nuget` to be installed. On mac:
- mono - on mac, `brew install mono`
- nuget - on mac, `brew install nuget` or you can [download nuget.exe](https://dist.nuget.org/win-x86-commandline/latest/nuget.exe)
- .NET build tools - can be loaded from [here](https://www.visualstudio.com/downloads/)
- nuget - download [nuget.exe](https://www.nuget.org/downloads)
- dotnet - follow [the instructions here](https://www.microsoft.com/net/core)
From the shell on mac, you can check all is ok by typing
Alternatively, you can install Visual Studio 2017 and make sure to check boxes with .NET Core SDK.
```bash
nuget
You also need to enable .NET Framework 3.5 support in Windows "Programs and Features".
If everything is ok, the following command will restore nuget packages, build Antlr for .NET Standard and .NET 3.5 and create nuget package:
```PS
msbuild /target:restore /target:rebuild /target:pack /property:Configuration=Release .\Antlr4.dotnet.sln /verbosity:minimal
```
This should display the nuget help.
This should display something like this:
**Creating and packaging the assembly**
```bash
$ cd runtime/CSharp/runtime/CSharp
$ ./build-nuget-package.sh
...
Build succeeded.
0 Warning(s)
0 Error(s)
Attempting to build package from 'Package.nuspec'.
Successfully created package '/path/to/antlr/.../Antlr4.Runtime.Standard.4.7.0.nupkg'.
```
Microsoft (R) Build Engine version 15.4.8.50001 for .NET Framework
Copyright (C) Microsoft Corporation. All rights reserved.
This should display: Successfully created package *<package-path>*
Alternately, you may want to build ANTLR using Xamarin Studio Community (free).
Restoring packages for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj...
Generating MSBuild file C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\obj\Antlr4.Runtime.dotnet.csproj.nuget.g.props.
Generating MSBuild file C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\obj\Antlr4.Runtime.dotnet.csproj.nuget.g.targets.
Restore completed in 427.62 ms for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj.
Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\netstandard1.3\Antlr4.Runtime.Standard.dll
Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\net35\Antlr4.Runtime.Standard.dll
Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.7.2.nupkg'.
```
**Publishing to NuGet**
You need to be a NuGet owner for "ANTLR 4 Standard Runtime"
As a registered NuGet user, you can then manually upload the package spec here (`runtime/CSharp/runtime/CSharp/Package.nuspec`): [https://www.nuget.org/packages/manage/upload](https://www.nuget.org/packages/manage/upload)
As a registered NuGet user, you can then manually upload the package here: [https://www.nuget.org/packages/manage/upload](https://www.nuget.org/packages/manage/upload)
Alternately, you can publish from the cmd line. You need to get your NuGet key from [https://www.nuget.org/account#](https://www.nuget.org/account#) and then from the cmd line, you can then type:
```bash
```cmd
nuget push Antlr4.Runtime.Standard.<version>.nupkg <your-key> -Source https://www.nuget.org/api/v2/package
```
**Creating DLLs**
```bash
cd ~/antlr/code/antlr4/runtime/CSharp/runtime/CSharp
# kill previous ones manually as "xbuild /t:Clean" didn't seem to do it
rm Antlr4.Runtime/bin/net20/Release/Antlr4.Runtime.dll
rm Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.dll
# build
xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj
# zip it up to get a version number on zip filename
zip --junk-paths /tmp/antlr-csharp-runtime-4.7.zip Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.Standard.dll
cp /tmp/antlr-csharp-runtime-4.7.zip ~/antlr/sites/website-antlr4/download
```
Move target to website
```bash
pushd ~/antlr/sites/website-antlr4/download
git add antlr-csharp-runtime-4.7.zip
git commit -a -m 'update C# runtime'
git push origin gh-pages
popd
```
Nuget packages are also accessible as artifacts of [AppVeyor builds](https://ci.appveyor.com/project/parrt/antlr4/build/artifacts).
### Python

View File

@ -10,6 +10,7 @@ import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.StreamVacuum;
@ -337,7 +338,9 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
}
public String execRecognizer() {
compile();
boolean success = compile();
assertTrue(success);
String output = execTest();
if ( output!=null && output.length()==0 ) {
output = null;
@ -354,6 +357,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
return false;
return true;
} catch(Exception e) {
e.printStackTrace(System.err);
return false;
}
}
@ -362,6 +366,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
try {
return buildDotnetProject();
} catch(Exception e) {
e.printStackTrace(System.err);
return false;
}
}
@ -390,10 +395,15 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
stdoutVacuum.join();
stderrVacuum.join();
// xbuild sends errors to output, so check exit code
boolean success = process.exitValue()==0;
int exitValue = process.exitValue();
boolean success = (exitValue == 0);
if ( !success ) {
this.stderrDuringParse = stdoutVacuum.toString();
System.err.println("buildProject stderrVacuum: "+ this.stderrDuringParse);
String stderrString = stderrVacuum.toString();
System.err.println("buildProject command: " + Utils.join(args, " "));
System.err.println("buildProject exitValue: " + exitValue);
System.err.println("buildProject stdout: " + stderrDuringParse);
System.err.println("buildProject stderr: " + stderrString);
}
return success;
}
@ -505,6 +515,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
runtimeProjPath
};
boolean success = runProcess(args, tmpdir);
assertTrue(success);
// restore project
args = new String[] {
@ -514,6 +525,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
"--no-dependencies"
};
success = runProcess(args, tmpdir);
assertTrue(success);
// build test
args = new String[] {
@ -525,6 +537,7 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
"--no-dependencies"
};
success = runProcess(args, tmpdir);
assertTrue(success);
}
catch(Exception e) {
e.printStackTrace(System.err);
@ -535,6 +548,10 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
}
private boolean runProcess(String[] args, String path) throws Exception {
return runProcess(args, path, 0);
}
private boolean runProcess(String[] args, String path, int retries) throws Exception {
ProcessBuilder pb = new ProcessBuilder(args);
pb.directory(new File(path));
Process process = pb.start();
@ -545,10 +562,28 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
process.waitFor();
stdoutVacuum.join();
stderrVacuum.join();
boolean success = process.exitValue()==0;
int exitValue = process.exitValue();
boolean success = (exitValue == 0);
if ( !success ) {
this.stderrDuringParse = stderrVacuum.toString();
System.err.println("runProcess stderrVacuum: "+ this.stderrDuringParse);
System.err.println("runProcess command: " + Utils.join(args, " "));
System.err.println("runProcess exitValue: " + exitValue);
System.err.println("runProcess stdoutVacuum: " + stdoutVacuum.toString());
System.err.println("runProcess stderrVacuum: " + stderrDuringParse);
}
if (exitValue == 132) {
// Retry after SIGILL. We are seeing this intermittently on
// macOS (issue #2078).
if (retries < 3) {
System.err.println("runProcess retrying; " + retries +
" retries so far");
return runProcess(args, path, retries + 1);
}
else {
System.err.println("runProcess giving up after " + retries +
" retries");
return false;
}
}
return success;
}
@ -577,9 +612,28 @@ public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTest
ProcessBuilder pb = new ProcessBuilder(args);
pb.directory(tmpdirFile);
Process process = pb.start();
StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
stdoutVacuum.start();
stderrVacuum.start();
process.waitFor();
stdoutVacuum.join();
stderrVacuum.join();
String writtenOutput = TestOutputReading.read(output);
this.stderrDuringParse = TestOutputReading.read(errorOutput);
int exitValue = process.exitValue();
String stdoutString = stdoutVacuum.toString().trim();
String stderrString = stderrVacuum.toString().trim();
if (exitValue != 0) {
System.err.println("execTest command: " + Utils.join(args, " "));
System.err.println("execTest exitValue: " + exitValue);
}
if (!stdoutString.isEmpty()) {
System.err.println("execTest stdoutVacuum: " + stdoutString);
}
if (!stderrString.isEmpty()) {
System.err.println("execTest stderrVacuum: " + stderrString);
}
return writtenOutput;
}
catch (Exception e) {

View File

@ -1,20 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Company>The ANTLR Organization</Company>
<Version>4.7.1</Version>
<TargetFramework>netstandard1.3</TargetFramework>
<DefineConstants>$(DefineConstants);DOTNETCORE;NET35PLUS;NET40PLUS;NET45PLUS</DefineConstants>
<Version>4.7.2</Version>
<NeutralLanguage>en-US</NeutralLanguage>
<TargetFrameworks>netstandard1.3;net35</TargetFrameworks>
<NoWarn>$(NoWarn);CS1591;CS1574;CS1580</NoWarn>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<AssemblyName>Antlr4.Runtime.Core</AssemblyName>
<AssemblyName>Antlr4.Runtime.Standard</AssemblyName>
<AssemblyOriginatorKeyFile>../../Antlr4.snk</AssemblyOriginatorKeyFile>
<SignAssembly>true</SignAssembly>
<PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign>
<PackageId>Antlr4.Runtime.Core</PackageId>
<Title>ANTLR 4 .NET Core Runtime</Title>
<PackageId>Antlr4.Runtime.Standard</PackageId>
<Title>ANTLR 4 .NET Standard Runtime</Title>
<Authors>Eric Vergnaud, Terence Parr, Sam Harwell</Authors>
<Description>The .NET Core C# ANTLR 4 runtime from the ANTLR Organization</Description>
<Summary>The runtime library for parsers generated by the C# target of the standard ANTLR 4 tool.</Summary>
<Copyright>Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.</Copyright>
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
<PackageLicenseUrl>https://github.com/antlr/antlr4/blob/master/LICENSE.txt</PackageLicenseUrl>
@ -32,6 +32,7 @@
<GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
<GenerateAssemblyFileVersionAttribute>false</GenerateAssemblyFileVersionAttribute>
<GenerateAssemblyInformationalVersionAttribute>false</GenerateAssemblyInformationalVersionAttribute>
<RootNamespace>Antlr4.Runtime</RootNamespace>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
@ -43,5 +44,10 @@
<Optimize>true</Optimize>
<OutputPath>lib\Release</OutputPath>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='netstandard1.3'">
<DefineConstants>DOTNETCORE;NET35PLUS;NET40PLUS;NET45PLUS</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net35'">
<DefineConstants>NET35PLUS</DefineConstants>
</PropertyGroup>
</Project>

View File

@ -1,25 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://schemas.microsoft.com/packaging/2011/08/nuspec.xsd">
<metadata>
<id>Antlr4.Runtime.Standard</id>
<version>4.7.1</version>
<language>en-us</language>
<title>ANTLR 4 Standard Runtime</title>
<description>The standard C# ANTLR 4 runtime from the ANTLR Organization</description>
<summary>The runtime library for parsers generated by the C# target of the standard ANTLR 4 tool.</summary>
<authors>Eric Vergnaud, Terence Parr, Sam Harwell</authors>
<owners>The ANTLR Organization</owners>
<releaseNotes>https://github.com/antlr/antlr4/releases</releaseNotes>
<requireLicenseAcceptance>true</requireLicenseAcceptance>
<licenseUrl>https://github.com/antlr/antlr4/blob/master/LICENSE.txt</licenseUrl>
<projectUrl>https://github.com/antlr/antlr4</projectUrl>
<iconUrl>https://raw.github.com/antlr/website-antlr4/master/images/icons/antlr.png</iconUrl>
<copyright>Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.</copyright>
<tags>antlr parsing grammar</tags>
<dependencies />
</metadata>
<files>
<file src="Antlr4.Runtime/lib/Release/Antlr4.Runtime.Standard.dll" target="lib/net35/"/>
<file src="Antlr4.Runtime/lib/Release/netstandard1.3/Antlr4.Runtime.Core.dll" target="lib/netstandard/"/>
</files>
</package>

View File

@ -1,14 +0,0 @@
#!/bin/sh
# Build a .NET 3.5 compatible DLL using mono
# This step can be done by the `dotnet` cli once https://github.com/Microsoft/msbuild/issues/1333 is resolved.
echo "Step 1: Building .NET 3.5 DLL"
xbuild /p:Configuration=Release Antlr4.mono.sln
# Build a .NET core DLL using the `dotnet` cli from microsoft
echo "Step 2: Building .NET Core DLL"
dotnet restore Antlr4.dotnet.sln
dotnet build -c Release Antlr4.dotnet.sln
echo "Step 3: Packaging both DLLs into a single nuget package"
nuget pack Package.nuspec

View File

@ -267,7 +267,7 @@ DefaultErrorStrategy.prototype.reportNoViableAlternative = function(recognizer,
if (e.startToken.type===Token.EOF) {
input = "<EOF>";
} else {
input = tokens.getText(new Interval(e.startToken, e.offendingToken));
input = tokens.getText(new Interval(e.startToken.tokenIndex, e.offendingToken.tokenIndex));
}
} else {
input = "<unknown input>";
@ -753,4 +753,4 @@ BailErrorStrategy.prototype.sync = function(recognizer) {
};
exports.BailErrorStrategy = BailErrorStrategy;
exports.DefaultErrorStrategy = DefaultErrorStrategy;
exports.DefaultErrorStrategy = DefaultErrorStrategy;