diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 59c6da0e1..d43c55be0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -162,6 +162,8 @@ jobs: run: ./download_v8.sh - name: SBG tests run: ./gradlew runSbgTests --stacktrace + - name: Metadata generator tests + run: ./gradlew runMetadataGeneratorTests --stacktrace - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules diff --git a/build.gradle b/build.gradle index 71f536ec2..f9bd5c264 100644 --- a/build.gradle +++ b/build.gradle @@ -454,6 +454,17 @@ task runSbgTests (type: Exec, dependsOn: 'runAstTests') { } } +task runMetadataGeneratorTests (type: Exec) { + doFirst { + workingDir "$TEST_APP_PATH" + if (isWinOs) { + commandLine "cmd", "/c", "gradlew", ":android-metadata-generator:test", "--warning-mode", "all" + } else { + commandLine "./gradlew", ":android-metadata-generator:test", "--warning-mode", "all" + } + } +} + def getRunTestsBuildArguments = { taskName -> def arguments = [] if (isWinOs) { @@ -474,6 +485,9 @@ def getRunTestsBuildArguments = { taskName -> if (selectedAbis != null) { arguments.add("-Pabis=${selectedAbis}") } + if (project.hasProperty("narrowMetadataFields")) { + arguments.add("-PnarrowMetadataFields") + } arguments += ["--warning-mode", "all"] return arguments } diff --git a/test-app/app/build.gradle b/test-app/app/build.gradle index d90f7ffe0..9e0245f29 100644 --- a/test-app/app/build.gradle +++ b/test-app/app/build.gradle @@ -95,6 +95,9 @@ def computeBuildToolsVersion = { -> def enableAnalytics = (project.hasProperty("gatherAnalyticsData") && project.gatherAnalyticsData == "true") def enableVerboseMDG = project.gradle.startParameter.logLevel.name() == 'DEBUG' +// For the runtime's own tests: writes every variable-width metadata field a byte wider +// than this app needs, the way only a very large app's metadata would otherwise be. +def widenMetadataFields = project.hasProperty("widenMetadataFields") def analyticsFilePath = "$rootDir/analytics/build-statistics.json" def analyticsCollector = project.ext.AnalyticsCollector.withOutputPath(analyticsFilePath) if (enableAnalytics) { @@ -998,6 +1001,8 @@ task buildMetadata(type: BuildToolTask) { } } + inputs.property("widenMetadataFields", widenMetadataFields) + outputs.files("$METADATA_OUT_PATH/treeNodeStream.dat", "$METADATA_OUT_PATH/treeStringsStream.dat", "$METADATA_OUT_PATH/treeValueStream.dat") workingDir "$BUILD_TOOLS_PATH" @@ -1083,6 +1088,10 @@ task buildMetadata(type: BuildToolTask) { paramz.add("verbose") } + if (widenMetadataFields) { + paramz.add("widen-fields") + } + args paramz.toArray() } } diff --git a/test-app/build-tools/android-metadata-generator/build.gradle b/test-app/build-tools/android-metadata-generator/build.gradle index 85ca0723e..def27d4da 100644 --- a/test-app/build-tools/android-metadata-generator/build.gradle +++ b/test-app/build-tools/android-metadata-generator/build.gradle @@ -51,6 +51,14 @@ compileKotlin { } } +// Without this the test sources target whatever JDK runs the build, and Kotlin refuses a +// target that differs from the Java one. +compileTestKotlin { + kotlinOptions { + jvmTarget = '17' + } +} + repositories { google() mavenCentral() diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/FileStreamWriter.java b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/FileStreamWriter.java index ac5c6b2bb..ffde2aba0 100644 --- a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/FileStreamWriter.java +++ b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/FileStreamWriter.java @@ -1,23 +1,33 @@ package com.telerik.metadata; +import java.io.BufferedOutputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; public class FileStreamWriter implements StreamWriter { - private final FileOutputStream out; + // The metadata is written a few bytes at a time, hundreds of thousands of times. + private static final int BUFFER_SIZE = 1 << 16; + + private final OutputStream out; + + // Counted here, since the file's own position lags behind what is still buffered. + // The stream is expected to be at the start of an empty file. + private int position; public FileStreamWriter(FileOutputStream out) { - this.out = out; + this.out = new BufferedOutputStream(out, BUFFER_SIZE); } @Override public int getPosition() throws IOException { - return (int) out.getChannel().position(); + return position; } @Override public void write(byte[] buffer) throws IOException { out.write(buffer); + position += buffer.length; } @Override @@ -33,5 +43,6 @@ public void close() throws IOException { @Override public void write(byte b) throws IOException { out.write(b); + position++; } } diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Generator.java b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Generator.java index 6a53cea27..4c39b72b6 100644 --- a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Generator.java +++ b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Generator.java @@ -20,6 +20,7 @@ public class Generator { private static final String VERBOSE_FLAG_NAME = "verbose"; private static final String SKIP_FLAG_NAME = "skip"; + private static final String WIDEN_FIELDS_FLAG_NAME = "widen-fields"; private static final String ANALYTICS_ARGUMENT_BEGINNING = "analyticsFilePath="; private static final String MDG_OUTPUT_DIR = "mdg-output-dir.txt"; private static final String MDG_JAVA_DEPENDENCIES = "mdg-java-dependencies.txt"; @@ -28,6 +29,7 @@ public class Generator { private static final String METADATA_JAVA_OUT = "mdg-java-out.txt"; private static boolean verbose_mode = false; + private static boolean widen_fields = false; /** * @param args arguments @@ -74,9 +76,9 @@ public static void main(String[] args) { if (verbose_mode) { FileOutputStream ods = new FileOutputStream(new File("metadata-debug.json")); FileStreamWriter outDebugStream = new FileStreamWriter(ods); - new Writer(outNodeStream, outValueStream, outStringsStream, outDebugStream).writeTree(root); + new Writer(outNodeStream, outValueStream, outStringsStream, outDebugStream, widen_fields).writeTree(root); } else { - new Writer(outNodeStream, outValueStream, outStringsStream).writeTree(root); + new Writer(outNodeStream, outValueStream, outStringsStream, null, widen_fields).writeTree(root); } } catch (Throwable ex) { System.err.println(String.format("Error executing Metadata Generator: %s", ex.getMessage())); @@ -93,6 +95,8 @@ private static void enableFlaggedFeatures(String[] args) { } else if (VERBOSE_FLAG_NAME.equals(arg)) { verbose_mode = true; MetadataFilterConsoleLogger.INSTANCE.setEnabled(true); + } else if (WIDEN_FIELDS_FLAG_NAME.equals(arg)) { + widen_fields = true; } else if (SKIP_FLAG_NAME.equals(arg)) { System.out.println("Skipping metadata generation: skip flag used."); exit(0); diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/TreeNode.java b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/TreeNode.java index d43b17f46..184aff52d 100644 --- a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/TreeNode.java +++ b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/TreeNode.java @@ -218,9 +218,9 @@ public void setName(String value) { private String _name; // - public short id; - public short firstChildId; - public short nextSiblingId; + public int id; + public int firstChildId; + public int nextSiblingId; // public byte nodeType; public int offsetName; diff --git a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Writer.java b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Writer.java index be9cb4d78..842b6d930 100644 --- a/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Writer.java +++ b/test-app/build-tools/android-metadata-generator/src/src/com/telerik/metadata/Writer.java @@ -7,9 +7,6 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -import java.nio.Buffer; -import java.nio.ByteOrder; -import java.nio.IntBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.HashMap; @@ -25,10 +22,14 @@ public class Writer { private final StreamWriter outDebugStream; // The bytes "NSM" and the format version. The runtime refuses any other value. - private static final int FORMAT = 'N' | ('S' << 8) | ('M' << 16) | (2 << 24); + private static final int FORMAT = 'N' | ('S' << 8) | ('M' << 16) | (3 << 24); - // A method record refers to its signature with 23 bits. - private static final int MAX_SIGNATURE_BYTES = 1 << 23; + // The runtime numbers the nodes it creates itself from here. + private static final int MAX_NODES = 1 << 30; + + // The widest a field can be. The runtime reads every field as this many bytes and + // masks off what is not the field's. + private static final int MAX_FIELD_BYTES = 4; private static final int SECTION_EXTENSION_FUNCTIONS = 1; private static final int SECTION_INSTANCE_METHODS = 1 << 1; @@ -48,23 +49,53 @@ public class Writer { private final HashMap signatureOffsets = new HashMap<>(); private final ByteArrayOutputStream signatures = new ByteArrayOutputStream(); + // How many bytes a node id, an offset into the strings and a signature reference take. + // Each is the fewest that hold the largest value this metadata needs, and the value + // stream's header tells the runtime what they are. + private int idBytes; + private int nameBytes; + private int signatureBytes; + + // Makes every one of those a byte wider than needed. The runtime's tests are built + // this way, so the widths only a very large app would get are the ones they cover. + private final boolean widenFields; + public Writer(StreamWriter outNodeStream, StreamWriter outValueStream, StreamWriter outStringsStream) { this(outNodeStream, outValueStream, outStringsStream, null); } public Writer(StreamWriter outNodeStream, StreamWriter outValueStream, StreamWriter outStringsStream, StreamWriter outDebugStream) { + this(outNodeStream, outValueStream, outStringsStream, outDebugStream, false); + } + public Writer(StreamWriter outNodeStream, StreamWriter outValueStream, + StreamWriter outStringsStream, StreamWriter outDebugStream, + boolean widenFields) { this.outNodeStream = outNodeStream; this.outValueStream = outValueStream; this.outStringsStream = outStringsStream; this.outDebugStream = outDebugStream; + this.widenFields = widenFields; + } + + private int fieldBytes(long largestValue) { + int bytes = 1; + while (bytes < MAX_FIELD_BYTES && (largestValue >>> (8 * bytes)) != 0) { + bytes++; + } + + return widenFields ? Math.min(MAX_FIELD_BYTES, bytes + 1) : bytes; + } + + private static void writeField(int value, int bytes, StreamWriter out) throws Exception { + for (int i = 0; i < bytes; i++) { + out.write((byte) ((value >>> (8 * i)) & 0xFF)); + } } private final static byte[] writeUniqueName_lenBuff = new byte[2]; private final static byte[] writeInt_buff = new byte[4]; - private final static byte[] writeTreeNodeId_buff = new byte[2]; private final static byte[] writeLength_lenBuff = new byte[2]; - private final static byte[] writeSignatureReference_buff = new byte[3]; private static int writeUniqueName(String name, HashMap uniqueStrings, @@ -94,20 +125,21 @@ private static void writeInt(int value, StreamWriter out) throws Exception { // One length byte, holding the count of ids less one, then the ids: the return type // followed by the parameter types. - private static ByteBuffer encodeSignature(MethodInfo mi) throws Exception { + private ByteBuffer encodeSignature(MethodInfo mi) throws Exception { int length = mi.signature.size(); if (length < 1 || length > 256) { throw new Exception("Method " + mi.name + " has a signature of " + length + " types, which the metadata cannot hold. Please report this issue."); } - byte[] record = new byte[1 + 2 * length]; + byte[] record = new byte[1 + idBytes * length]; record[0] = (byte) (length - 1); for (int i = 0; i < length; i++) { TreeNode type = mi.signature.get(i); - int id = (type == null) ? 0 : Short.toUnsignedInt(type.id); - record[1 + 2 * i] = (byte) (id & 0xFF); - record[2 + 2 * i] = (byte) ((id >> 8) & 0xFF); + int id = (type == null) ? 0 : type.id; + for (int b = 0; b < idBytes; b++) { + record[1 + idBytes * i + b] = (byte) ((id >>> (8 * b)) & 0xFF); + } } return ByteBuffer.wrap(record); @@ -150,11 +182,6 @@ private void collectSignatures(TreeNode root) throws Exception { d.addAll(n.children); } - if (signatures.size() >= MAX_SIGNATURE_BYTES) { - throw new Exception("Metadata is too big: the method signatures take " + signatures.size() - + " bytes and the format can address " + MAX_SIGNATURE_BYTES - + ". Please report this issue."); - } } // An array's element node takes its value from the node it stands for, so it has no @@ -166,30 +193,20 @@ private static boolean hasClassValue(TreeNode n) { || ((n.nodeType & TreeNode.Interface) == TreeNode.Interface)); } - // The name, then three bytes: the signature's offset in the signature region, shifted - // left by one, with the low bit set when the method is resolved. + // The name, then the signature reference: the signature's offset in the signature + // region, shifted left by one, with the low bit set when the method is resolved. private void writeMethodInfo(MethodInfo mi, HashMap uniqueStrings, StreamWriter outValueStream) throws Exception { - int pos = uniqueStrings.get(mi.name); - writeInt(pos, outValueStream); + writeField(uniqueStrings.get(mi.name), nameBytes, outValueStream); int reference = (signatureOffsets.get(encodeSignature(mi)) << 1) | (mi.isResolved ? 1 : 0); - writeSignatureReference_buff[0] = (byte) (reference & 0xFF); - writeSignatureReference_buff[1] = (byte) ((reference >> 8) & 0xFF); - writeSignatureReference_buff[2] = (byte) ((reference >> 16) & 0xFF); - outValueStream.write(writeSignatureReference_buff); + writeField(reference, signatureBytes, outValueStream); } - private static void writeTreeNodeId(TreeNode node, StreamWriter out) + private void writeTreeNodeId(TreeNode node, StreamWriter out) throws Exception { - if (node == null) { - writeTreeNodeId_buff[0] = writeTreeNodeId_buff[1] = 0; - } else { - writeTreeNodeId_buff[0] = (byte) (node.id & 0xFF); - writeTreeNodeId_buff[1] = (byte) ((node.id >> 8) & 0xFF); - } - out.write(writeTreeNodeId_buff); + writeField((node == null) ? 0 : node.id, idBytes, out); } private static int writeLength(int length, StreamWriter out) @@ -242,7 +259,7 @@ public void writeClassValue(StreamWriter writer, if (!instanceFields.isEmpty()) { writeLength(instanceFields.size(), writer); for (FieldInfo fi : instanceFields) { - writeInt(stringsMap.get(fi.name), writer); + writeField(stringsMap.get(fi.name), nameBytes, writer); writeTreeNodeId(fi.valueType, writer); writer.write((byte) (fi.isFinalType ? FIELD_FINAL : 0)); } @@ -251,7 +268,7 @@ public void writeClassValue(StreamWriter writer, if (!propertyInfos.isEmpty()) { writeLength(propertyInfos.size(), writer); for (TreeNode.PropertyInfo propertyInfo : propertyInfos) { - writeInt(stringsMap.get(propertyInfo.getPropertyName()), writer); + writeField(stringsMap.get(propertyInfo.getPropertyName()), nameBytes, writer); Optional propertyGetter = propertyInfo.getGetterMethod(); Optional propertySetter = propertyInfo.getSetterMethod(); @@ -283,7 +300,7 @@ public void writeClassValue(StreamWriter writer, // inherited ones name their declaring type. boolean hasDeclaringType = fi.declaringType != n; - writeInt(stringsMap.get(fi.name), writer); + writeField(stringsMap.get(fi.name), nameBytes, writer); writeTreeNodeId(fi.valueType, writer); writer.write((byte) ((fi.isFinalType ? FIELD_FINAL : 0) | (hasDeclaringType ? FIELD_HAS_DECLARING_TYPE : 0))); @@ -295,7 +312,7 @@ public void writeClassValue(StreamWriter writer, } public void writeTree(TreeNode root) throws Exception { - short curId = 0; + int curId = 0; ArrayDeque d = new ArrayDeque<>(); @@ -312,8 +329,7 @@ public void writeTree(TreeNode root) throws Exception { d.push(root); while (!d.isEmpty()) { TreeNode n = d.pollFirst(); - if (Short.toUnsignedInt((short)(curId + 1)) < Short.toUnsignedInt(curId)) { - // we have overflowed our maximum (16 bit) metadata size + if (curId == MAX_NODES) { throw new Exception("Metadata is too big and has overflown our current limit, please report this issue"); } n.id = n.firstChildId = n.nextSiblingId = curId++; @@ -390,14 +406,23 @@ public void writeTree(TreeNode root) throws Exception { d.addAll(n.children); } + idBytes = fieldBytes(curId - 1); + nameBytes = fieldBytes(outStringsStream.getPosition() - 1); + outStringsStream.flush(); outStringsStream.close(); - // The value stream opens with the format, the size of the signature region and - // the region itself. No value starts at offset 0, which a package node uses to - // say it has none. collectSignatures(root); + signatureBytes = fieldBytes(((long) Math.max(signatures.size() - 1, 0) << 1) | 1); + + // The value stream opens with the format, the three field widths, the size of the + // signature region and the region itself. No value starts at offset 0, which a + // package node uses to say it has none. writeInt(FORMAT, outValueStream); + outValueStream.write((byte) idBytes); + outValueStream.write((byte) nameBytes); + outValueStream.write((byte) signatureBytes); + outValueStream.write((byte) 0); writeInt(signatures.size(), outValueStream); outValueStream.write(signatures.toByteArray()); @@ -434,6 +459,9 @@ public void writeTree(TreeNode root) throws Exception { d.addAll(n.children); } + // Room for the runtime to read the stream's last field at full width. + outValueStream.write(new byte[MAX_FIELD_BYTES]); + outValueStream.flush(); outValueStream.close(); @@ -442,7 +470,7 @@ public void writeTree(TreeNode root) throws Exception { TreeNode n = d.pollFirst(); if (n.arrayElement != null) { - n.offsetValue = array_offset + Short.toUnsignedInt(n.arrayElement.id); + n.offsetValue = array_offset + n.arrayElement.id; } if (!n.children.isEmpty()) { @@ -457,23 +485,17 @@ public void writeTree(TreeNode root) throws Exception { d.add(n.children.get(i)); } } - int[] nodeData = new int[3]; - - ByteBuffer byteBuffer = ByteBuffer.allocate(nodeData.length * 4); - byteBuffer.order(ByteOrder.LITTLE_ENDIAN); - IntBuffer intBuffer = byteBuffer.asIntBuffer(); - + // A node record: the ids of its first child and next sibling, its name and the + // offset of its value, which is always four bytes because its top range marks + // arrays. d.push(root); while (!d.isEmpty()) { TreeNode n = d.pollFirst(); - nodeData[0] = (n.firstChildId & 0xFFFF) | (n.nextSiblingId << 16); - nodeData[1] = n.offsetName; - nodeData[2] = n.offsetValue; - - ((Buffer)intBuffer).clear(); - intBuffer.put(nodeData); - outNodeStream.write(byteBuffer.array()); + writeField(n.firstChildId, idBytes, outNodeStream); + writeField(n.nextSiblingId, idBytes, outNodeStream); + writeField(n.offsetName, nameBytes, outNodeStream); + writeInt(n.offsetValue, outNodeStream); d.addAll(n.children); } @@ -487,9 +509,9 @@ public void writeTree(TreeNode root) throws Exception { while (!d.isEmpty()) { TreeNode n = d.pollFirst(); JsonObject obj = new JsonObject(); - obj.addProperty("id", Short.toUnsignedInt(n.id)); - obj.addProperty("nextSiblingId", Short.toUnsignedInt(n.nextSiblingId)); - obj.addProperty("firstChildId", Short.toUnsignedInt(n.firstChildId)); + obj.addProperty("id", n.id); + obj.addProperty("nextSiblingId", n.nextSiblingId); + obj.addProperty("firstChildId", n.firstChildId); obj.addProperty("offsetName", Integer.toUnsignedLong(n.offsetName)); obj.addProperty("offsetValue", Integer.toUnsignedLong(n.offsetValue)); obj.addProperty("name", n.getName()); diff --git a/test-app/build-tools/android-metadata-generator/src/test/java/com/telerik/metadata/WriterFormatTest.kt b/test-app/build-tools/android-metadata-generator/src/test/java/com/telerik/metadata/WriterFormatTest.kt new file mode 100644 index 000000000..28db30abd --- /dev/null +++ b/test-app/build-tools/android-metadata-generator/src/test/java/com/telerik/metadata/WriterFormatTest.kt @@ -0,0 +1,515 @@ +package com.telerik.metadata + +import com.telerik.metadata.TreeNode.FieldInfo +import com.telerik.metadata.TreeNode.MethodInfo +import com.telerik.metadata.TreeNode.PropertyInfo +import com.telerik.metadata.parsing.NativeMethodDescriptor +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Test +import java.io.ByteArrayOutputStream +import java.lang.reflect.Proxy + +/** + * Writes a small hand-built tree and reads the three streams back with a parser + * that knows only the format, the way the runtime's MetadataReader does. The + * parser fails unless every byte of the value stream belongs to exactly one + * record, so a field the writer and the format disagree on shows up here. + */ +class WriterFormatTest { + + @Test + fun `a small tree gets the narrowest fields that hold it`() { + val written = write(Fixture(padded = false), widenFields = false) + + assertEquals(1, written.idBytes) + assertMinimalWidths(written, widened = false) + assertMatches(written) + } + + @Test + fun `more nodes and longer names get wider ids and name offsets`() { + val written = write(Fixture(padded = true), widenFields = false) + + assertEquals(2, written.idBytes) + assertEquals(3, written.nameBytes) + assertMinimalWidths(written, widened = false) + assertMatches(written) + } + + @Test + fun `widening adds a byte to every field width`() { + val written = write(Fixture(padded = true), widenFields = true) + + assertEquals(3, written.idBytes) + assertEquals(4, written.nameBytes) + assertMinimalWidths(written, widened = true) + assertMatches(written) + } + + @Test + fun `a signature of more than 256 types is refused`() { + val fixture = Fixture(padded = false) + fixture.wide.addStaticFunction( + fixture.method("tooMany", List(257) { TreeNode.INTEGER }, declaringType = fixture.wide)) + + try { + write(fixture, widenFields = false) + fail("expected the writer to refuse a 257-type signature") + } catch (e: Exception) { + assertTrue(e.message, e.message!!.contains("257")) + } + } + + private fun assertMinimalWidths(written: Written, widened: Boolean) { + fun bytesFor(largest: Long): Int { + var bytes = 1 + while (bytes < 4 && (largest ushr (8 * bytes)) != 0L) bytes++ + return if (widened) minOf(4, bytes + 1) else bytes + } + + assertEquals(bytesFor(written.nodes.size - 1L), written.idBytes) + assertEquals(bytesFor(written.stringsLength - 1L), written.nameBytes) + assertEquals(bytesFor(((written.signaturesLength - 1L) shl 1) or 1), written.signatureBytes) + } + + private fun assertMatches(written: Written) { + val f = written.fixture + + // Tree links: a node with no child, or no further sibling, names itself. + assertEquals(f.root.id, written.nodes[f.root.id].nextSiblingId) + f.allNodes().forEach { node -> + val parsed = written.nodes[node.id] + assertEquals(node.name, parsed.name) + assertEquals(node.children.firstOrNull()?.id ?: node.id, parsed.firstChildId) + node.children.forEachIndexed { index, child -> + val next = node.children.getOrNull(index + 1) + assertEquals(next?.id ?: child.id, written.nodes[child.id].nextSiblingId) + } + } + + // Every distinct signature is stored once, and methods that share one refer to + // the same record. + val distinctSignatures = f.methods.map { it.signature.map { type -> type?.id ?: 0 } }.toSet() + assertEquals(distinctSignatures.size, written.signatureCount) + val hashCode = written.instanceMethod(f.obj, "hashCode") + val size = written.instanceMethod(f.base, "size") + assertEquals(listOf(TreeNode.INTEGER.id), hashCode.signature) + assertEquals(hashCode.signatureOffset, size.signatureOffset) + assertNotEquals(hashCode.signatureOffset, written.instanceMethod(f.obj, "toString").signatureOffset) + + // A void return type is id 0, and the resolved flag travels with the reference. + assertEquals(listOf(0), written.instanceMethod(f.obj, "notify").signature) + assertFalse(written.instanceMethod(f.obj, "notify").resolved) + val equals = written.instanceMethod(f.obj, "equals") + assertTrue(equals.resolved) + assertEquals(listOf(TreeNode.BOOLEAN.id, f.obj.id), equals.signature) + + // The longest signature the format holds. + val many = written.classOf(f.wide).staticMethods.single() + assertEquals(256, many.signature.size) + assertEquals(f.wideTypes.map { it.id }, many.signature) + assertEquals(f.wide.id, many.declaringId) + + // Base classes. + assertEquals(f.obj.id, written.classOf(f.base).baseId) + assertEquals(f.base.id, written.classOf(f.derived).baseId) + + // Only the sections a class has are there. + assertEquals(SECTION_STATIC_METHODS, written.classOf(f.staticOnly).sections) + assertEquals(SECTION_INSTANCE_METHODS, written.classOf(f.iface).sections) + assertEquals(SECTION_PROPERTIES, written.classOf(f.props).sections) + assertEquals(SECTION_EXTENSION_FUNCTIONS or SECTION_INSTANCE_METHODS or SECTION_STATIC_METHODS + or SECTION_STATIC_FIELDS, written.classOf(f.derived).sections) + + // Instance fields. + val baseFields = written.classOf(f.base).instanceFields.associateBy { it.name } + assertEquals(TreeNode.INTEGER.id, baseFields.getValue("count").typeId) + assertEquals(0, baseFields.getValue("count").flags) + assertEquals(f.string.id, baseFields.getValue("label").typeId) + assertEquals(FIELD_FINAL, baseFields.getValue("label").flags) + + // A static field names its declaring type only when another class declares it. + val own = written.classOf(f.base).staticFields.single() + assertEquals("OWN", own.name) + assertEquals(FIELD_FINAL, own.flags) + assertNull(own.declaringId) + val derivedFields = written.classOf(f.derived).staticFields.associateBy { it.name } + assertEquals(FIELD_FINAL or FIELD_HAS_DECLARING_TYPE, derivedFields.getValue("OWN").flags) + assertEquals(f.base.id, derivedFields.getValue("OWN").declaringId) + assertEquals(0, derivedFields.getValue("LOCAL").flags) + assertNull(derivedFields.getValue("LOCAL").declaringId) + + // Extension functions and static methods carry their declaring type. + val extension = written.classOf(f.derived).extensionFunctions.single() + assertEquals("ext", extension.name) + assertEquals(f.staticOnly.id, extension.declaringId) + assertEquals(listOf(TreeNode.INTEGER.id, f.derived.id), extension.signature) + assertEquals(f.derived.id, written.classOf(f.derived).staticMethods.single().declaringId) + + // Property accessors. + val properties = written.classOf(f.props).properties.associateBy { it.name } + assertEquals("getReadOnly", properties.getValue("readOnly").getter!!.name) + assertNull(properties.getValue("readOnly").setter) + assertNull(properties.getValue("writeOnly").getter) + assertEquals("setWriteOnly", properties.getValue("writeOnly").setter!!.name) + assertEquals("getBoth", properties.getValue("both").getter!!.name) + assertEquals("setBoth", properties.getValue("both").setter!!.name) + + // An array node, and an element node that stands for another node and has no + // value of its own. + assertEquals(ARRAY_OFFSET, written.nodes[f.arrays.id].valueOffset) + assertEquals(ARRAY_OFFSET + f.obj.id, written.nodes[f.objectArrayElement.id].valueOffset) + assertNull(written.nodes[f.objectArrayElement.id].classValue) + } + + // ---- the tree under test + + private class Fixture(padded: Boolean) { + val root: TreeNode = TreeNode.getRoot() + val methods = ArrayList() + + val obj: TreeNode + val string: TreeNode + val base: TreeNode + val derived: TreeNode + val staticOnly: TreeNode + val iface: TreeNode + val props: TreeNode + val wide: TreeNode + val arrays: TreeNode + val objectArrayElement: TreeNode + val wideTypes: List + + init { + if (padded) { + // Enough nodes for two-byte ids, and enough names for three-byte offsets. + val padding = root.createChild("padding") + repeat(300) { padding.createChild("p$it" + "x".repeat(250)) } + } + + val lang = root.createChild("java").createChild("lang") + obj = type(lang, "Object", TreeNode.Class, null) + string = type(lang, "String", TreeNode.Class, obj) + + val test = root.createChild("test") + base = type(test, "Base", TreeNode.Class, obj) + derived = type(test, "Derived", TreeNode.Class, base) + staticOnly = type(test, "StaticOnly", (TreeNode.Class.toInt() or TreeNode.Static.toInt()).toByte(), obj) + iface = type(test, "Iface", TreeNode.Interface, null) + props = type(test, "Props", TreeNode.Class, obj) + wide = type(test, "Wide", TreeNode.Class, obj) + + obj.addInstanceMethod(method("hashCode", listOf(TreeNode.INTEGER))) + obj.addInstanceMethod(method("equals", listOf(TreeNode.BOOLEAN, obj), resolved = true)) + obj.addInstanceMethod(method("toString", listOf(string))) + obj.addInstanceMethod(method("notify", listOf(null))) + + base.addInstanceMethod(method("size", listOf(TreeNode.INTEGER))) + base.addInstanceField(field("count", TreeNode.INTEGER, base, isFinal = false)) + base.addInstanceField(field("label", string, base, isFinal = true)) + base.addStaticField(field("OWN", TreeNode.INTEGER, base, isFinal = true)) + + derived.addInstanceMethod(method("get", listOf(obj, TreeNode.INTEGER))) + derived.addStaticFunction(method("create", listOf(derived), declaringType = derived)) + derived.addExtensionFunction(method("ext", listOf(TreeNode.INTEGER, derived), declaringType = staticOnly)) + derived.addStaticField(field("OWN", TreeNode.INTEGER, base, isFinal = true)) + derived.addStaticField(field("LOCAL", string, derived, isFinal = false)) + + staticOnly.addStaticFunction( + method("of", listOf(TreeNode.INTEGER, TreeNode.INTEGER), declaringType = staticOnly)) + + iface.addInstanceMethod(method("run", listOf(null))) + + props.addProperty(PropertyInfo("readOnly", method("getReadOnly", listOf(TreeNode.INTEGER)), null)) + props.addProperty(PropertyInfo("writeOnly", null, method("setWriteOnly", listOf(null, TreeNode.INTEGER)))) + props.addProperty(PropertyInfo("both", method("getBoth", listOf(string)), + method("setBoth", listOf(null, string)))) + + wideTypes = List(256) { if (it % 3 == 0) obj else TreeNode.INTEGER } + wide.addStaticFunction(method("many", wideTypes, declaringType = wide)) + + arrays = root.createChild("[") + arrays.nodeType = TreeNode.Array + objectArrayElement = arrays.createChild("Object") + objectArrayElement.nodeType = TreeNode.Class + objectArrayElement.arrayElement = obj + } + + fun allNodes(): List { + val all = ArrayList() + val queue = ArrayDeque(listOf(root)) + while (queue.isNotEmpty()) { + val node = queue.removeFirst() + all.add(node) + queue.addAll(node.children) + } + return all + } + + fun method(name: String, types: List, resolved: Boolean = false, + declaringType: TreeNode? = null): MethodInfo { + val signatureText = types.joinToString(",") { it?.name ?: "V" } + val descriptor = Proxy.newProxyInstance( + NativeMethodDescriptor::class.java.classLoader, + arrayOf(NativeMethodDescriptor::class.java)) { _, called, _ -> + when (called.name) { + "getName" -> name + "getSignature" -> signatureText + else -> throw UnsupportedOperationException(called.name) + } + } as NativeMethodDescriptor + + val info = MethodInfo(descriptor) + info.signature.addAll(types) + info.isResolved = resolved + info.declaringType = declaringType + methods.add(info) + return info + } + + private fun type(parent: TreeNode, name: String, nodeType: Byte, baseClass: TreeNode?): TreeNode { + val node = parent.createChild(name) + node.nodeType = nodeType + node.baseClassNode = baseClass + return node + } + + private fun field(name: String, valueType: TreeNode, declaringType: TreeNode, isFinal: Boolean): FieldInfo { + val info = FieldInfo(name) + info.valueType = valueType + info.declaringType = declaringType + info.isFinalType = isFinal + return info + } + } + + // ---- writing, and reading back from the format alone + + private class MemoryStream : StreamWriter { + private val out = ByteArrayOutputStream() + + override fun getPosition(): Int = out.size() + + override fun write(b: Byte) { + out.write(b.toInt()) + } + + override fun write(buffer: ByteArray) { + out.write(buffer) + } + + override fun flush() {} + + override fun close() {} + + fun bytes(): ByteArray = out.toByteArray() + } + + private class ParsedMethod(val name: String, val resolved: Boolean, val signatureOffset: Int, + val signature: List, val declaringId: Int?) + + private class ParsedField(val name: String, val typeId: Int, val flags: Int, val declaringId: Int?) + + private class ParsedProperty(val name: String, val getter: ParsedMethod?, val setter: ParsedMethod?) + + private class ParsedClass(val type: Int, val baseId: Int, val sections: Int, + val extensionFunctions: List, + val instanceMethods: List, + val instanceFields: List, + val properties: List, + val staticMethods: List, + val staticFields: List) + + private class ParsedNode(val name: String, val firstChildId: Int, val nextSiblingId: Int, + val valueOffset: Long, val classValue: ParsedClass?) + + private class Written(val fixture: Fixture, val idBytes: Int, val nameBytes: Int, val signatureBytes: Int, + val stringsLength: Int, val signaturesLength: Int, val signatureCount: Int, + val nodes: List) { + fun classOf(node: TreeNode): ParsedClass = nodes[node.id].classValue!! + + fun instanceMethod(node: TreeNode, name: String): ParsedMethod = + classOf(node).instanceMethods.single { it.name == name } + } + + private fun write(fixture: Fixture, widenFields: Boolean): Written { + val nodeStream = MemoryStream() + val valueStream = MemoryStream() + val stringsStream = MemoryStream() + Writer(nodeStream, valueStream, stringsStream, null, widenFields).writeTree(fixture.root) + + return parse(fixture, nodeStream.bytes(), valueStream.bytes(), stringsStream.bytes()) + } + + private fun parse(fixture: Fixture, nodes: ByteArray, values: ByteArray, strings: ByteArray): Written { + fun field(buffer: ByteArray, position: Int, width: Int): Long { + var value = 0L + for (i in 0 until width) { + value = value or ((buffer[position + i].toLong() and 0xFF) shl (8 * i)) + } + return value + } + + assertEquals(FORMAT, field(values, 0, 4)) + val idBytes = values[4].toInt() + val nameBytes = values[5].toInt() + val signatureBytes = values[6].toInt() + assertEquals(0, values[7].toInt()) + val signaturesLength = field(values, 8, 4).toInt() + + val used = BooleanArray(values.size) + fun mark(start: Int, end: Int, what: String) { + for (i in start until end) { + assertFalse("byte $i of the value stream is used twice ($what)", used[i]) + used[i] = true + } + } + mark(0, HEADER_SIZE, "header") + assertArrayEquals(ByteArray(PADDING), values.copyOfRange(values.size - PADDING, values.size)) + mark(values.size - PADDING, values.size, "padding") + + val nameStarts = HashMap() + var position = 0 + while (position < strings.size) { + val length = field(strings, position, 2).toInt() + nameStarts[position] = String(strings, position + 2, length, Charsets.UTF_8) + position += 2 + length + } + assertEquals(strings.size, position) + fun name(buffer: ByteArray, at: Int): String = + nameStarts[field(buffer, at, nameBytes).toInt()] ?: error("no name starts at that offset") + + val nodeRecordSize = 2 * idBytes + nameBytes + 4 + assertEquals(0, nodes.size % nodeRecordSize) + val nodeCount = nodes.size / nodeRecordSize + fun id(buffer: ByteArray, at: Int): Int { + val value = field(buffer, at, idBytes).toInt() + assertTrue("id $value is not a node", value in 0 until nodeCount) + return value + } + + // The signature region: records back to back, none of them twice. + val signatures = HashMap>() + position = HEADER_SIZE + while (position < HEADER_SIZE + signaturesLength) { + val length = (values[position].toInt() and 0xFF) + 1 + val types = List(length) { id(values, position + 1 + idBytes * it) } + signatures[position - HEADER_SIZE] = types + position += 1 + idBytes * length + } + assertEquals(HEADER_SIZE + signaturesLength, position) + assertEquals(signatures.size, signatures.values.toSet().size) + mark(HEADER_SIZE, HEADER_SIZE + signaturesLength, "signatures") + + val referenced = HashSet() + var cursor = 0 + fun method(declaring: Boolean): ParsedMethod { + val methodName = name(values, cursor) + val reference = field(values, cursor + nameBytes, signatureBytes).toInt() + val offset = reference ushr 1 + referenced.add(offset) + cursor += nameBytes + signatureBytes + val declaringId = if (declaring) id(values, cursor).also { cursor += idBytes } else null + return ParsedMethod(methodName, (reference and 1) == 1, offset, + signatures[offset] ?: error("no signature starts at $offset"), declaringId) + } + + fun count(): Int { + val entries = field(values, cursor, 2).toInt() + assertTrue("a section that is present has entries", entries > 0) + cursor += 2 + return entries + } + + val parsedNodes = List(nodeCount) { index -> + val record = index * nodeRecordSize + val valueOffset = field(nodes, record + 2 * idBytes + nameBytes, 4) + + var classValue: ParsedClass? = null + if (valueOffset in 1 until ARRAY_OFFSET) { + val start = valueOffset.toInt() + val type = values[start].toInt() and 0xFF + if ((type and PRIMITIVE) != 0) { + mark(start, start + 1, "primitive") + } else { + val baseId = id(values, start + 1) + val sections = values[start + 1 + idBytes].toInt() and 0xFF + assertEquals(0, sections ushr 6) + cursor = start + 1 + idBytes + 1 + + val extensionFunctions = if ((sections and SECTION_EXTENSION_FUNCTIONS) != 0) + List(count()) { method(declaring = true) } else emptyList() + val instanceMethods = if ((sections and SECTION_INSTANCE_METHODS) != 0) + List(count()) { method(declaring = false) } else emptyList() + val instanceFields = if ((sections and SECTION_INSTANCE_FIELDS) != 0) + List(count()) { + val parsed = ParsedField(name(values, cursor), id(values, cursor + nameBytes), + values[cursor + nameBytes + idBytes].toInt(), null) + cursor += nameBytes + idBytes + 1 + parsed + } else emptyList() + val properties = if ((sections and SECTION_PROPERTIES) != 0) + List(count()) { + val propertyName = name(values, cursor) + val accessors = values[cursor + nameBytes].toInt() + assertEquals(0, accessors ushr 2) + cursor += nameBytes + 1 + val getter = if ((accessors and 1) != 0) method(declaring = false) else null + val setter = if ((accessors and 2) != 0) method(declaring = false) else null + ParsedProperty(propertyName, getter, setter) + } else emptyList() + val staticMethods = if ((sections and SECTION_STATIC_METHODS) != 0) + List(count()) { method(declaring = true) } else emptyList() + val staticFields = if ((sections and SECTION_STATIC_FIELDS) != 0) + List(count()) { + val fieldName = name(values, cursor) + val typeId = id(values, cursor + nameBytes) + val flags = values[cursor + nameBytes + idBytes].toInt() + assertEquals(0, flags ushr 2) + cursor += nameBytes + idBytes + 1 + val declaringId = if ((flags and FIELD_HAS_DECLARING_TYPE) != 0) + id(values, cursor).also { cursor += idBytes } else null + ParsedField(fieldName, typeId, flags, declaringId) + } else emptyList() + + mark(start, cursor, "class value") + classValue = ParsedClass(type, baseId, sections, extensionFunctions, instanceMethods, + instanceFields, properties, staticMethods, staticFields) + } + } + + ParsedNode(name(nodes, record + 2 * idBytes), id(nodes, record), id(nodes, record + idBytes), + valueOffset, classValue) + } + + assertEquals("bytes of the value stream that belong to nothing", 0, used.count { !it }) + assertEquals("signatures nothing refers to", signatures.keys, referenced) + + return Written(fixture, idBytes, nameBytes, signatureBytes, strings.size, signaturesLength, + signatures.size, parsedNodes) + } + + private companion object { + const val FORMAT = ('N'.code or ('S'.code shl 8) or ('M'.code shl 16) or (3 shl 24)).toLong() + const val HEADER_SIZE = 12 + const val PADDING = 4 + const val ARRAY_OFFSET = Int.MAX_VALUE.toLong() + const val PRIMITIVE = 1 shl 4 + + const val SECTION_EXTENSION_FUNCTIONS = 1 + const val SECTION_INSTANCE_METHODS = 1 shl 1 + const val SECTION_INSTANCE_FIELDS = 1 shl 2 + const val SECTION_PROPERTIES = 1 shl 3 + const val SECTION_STATIC_METHODS = 1 shl 4 + const val SECTION_STATIC_FIELDS = 1 shl 5 + + const val FIELD_FINAL = 1 + const val FIELD_HAS_DECLARING_TYPE = 1 shl 1 + } +} diff --git a/test-app/runtests.gradle b/test-app/runtests.gradle index d8944745c..445b69eec 100644 --- a/test-app/runtests.gradle +++ b/test-app/runtests.gradle @@ -74,6 +74,12 @@ def getBuildArguments = { -> } arguments.add("-PuseKotlin=true") + // Ordinary apps get the narrowest metadata fields they can; the suite runs on the + // wider ones, which nothing else would exercise. -PnarrowMetadataFields runs it on + // the ones that ship. + if (!project.hasProperty("narrowMetadataFields")) { + arguments.add("-PwidenMetadataFields") + } return arguments } diff --git a/test-app/runtime/src/main/cpp/MetadataEntry.cpp b/test-app/runtime/src/main/cpp/MetadataEntry.cpp index ca77470c5..a10324489 100644 --- a/test-app/runtime/src/main/cpp/MetadataEntry.cpp +++ b/test-app/runtime/src/main/cpp/MetadataEntry.cpp @@ -7,7 +7,7 @@ using namespace tns; MetadataEntry::MetadataEntry(MetadataTreeNode *m_treeNode, NodeType nodeType) : treeNode(m_treeNode), type(nodeType), isExtensionFunction(false), isStatic(false), - isTypeMember(false), memberId(nullptr), clazz(nullptr), mi(nullptr),fi(nullptr), sfi(nullptr), + isTypeMember(false), memberId(nullptr), clazz(nullptr), mi(nullptr), field(nullptr), retType(MethodReturnType::Unknown), paramCount(-1), isFinal(false), isResolved(false), retTypeParsed(false), isFinalSet(false), isResolvedSet(false) {} @@ -17,10 +17,8 @@ std::string &MetadataEntry::getName() { auto reader = MetadataNode::getMetadataReader(); - if (type == NodeType::Field) { - name = reader->ReadName(fi->nameOffset); - } else if (type == NodeType::StaticField) { - name = reader->ReadName(sfi->nameOffset); + if (type == NodeType::Field || type == NodeType::StaticField) { + name = reader->ReadName(reader->FieldNameOffset(field)); } else if (type == NodeType::Method) { name = mi.GetName(); } @@ -33,10 +31,8 @@ std::string &MetadataEntry::getSig() { auto reader = MetadataNode::getMetadataReader(); - if (type == NodeType::Field) { - sig = reader->ReadTypeName(fi->nodeId); - } else if (type == NodeType::StaticField) { - sig = reader->ReadTypeName(sfi->nodeId); + if (type == NodeType::Field || type == NodeType::StaticField) { + sig = reader->ReadTypeName(reader->FieldTypeId(field)); } else if (type == NodeType::Method) { auto sigLength = mi.GetSignatureLength(); if (sigLength > 0) @@ -82,8 +78,8 @@ std::string &MetadataEntry::getDeclaringType() { auto reader = MetadataNode::getMetadataReader(); if (type == NodeType::StaticField) { - declaringType = (sfi->flags & FieldInfo::HAS_DECLARING_TYPE) - ? reader->ReadTypeName(sfi->declaringType) + declaringType = (reader->FieldFlags(field) & FieldInfo::HAS_DECLARING_TYPE) + ? reader->ReadTypeName(reader->FieldDeclaringTypeId(field)) : reader->ReadTypeName(treeNode); } else if (type == NodeType::Method && isStatic) { declaringType = mi.GetDeclaringType(); @@ -112,10 +108,9 @@ int MetadataEntry::getParamCount() { bool MetadataEntry::getIsFinal() { if (isFinalSet) return isFinal; - if (type == NodeType::Field) { - isFinal = (fi->flags & FieldInfo::FINAL) != 0; - } else if (type == NodeType::StaticField) { - isFinal = (sfi->flags & FieldInfo::FINAL) != 0; + if (type == NodeType::Field || type == NodeType::StaticField) { + auto reader = MetadataNode::getMetadataReader(); + isFinal = (reader->FieldFlags(field) & FieldInfo::FINAL) != 0; } isFinalSet = true; diff --git a/test-app/runtime/src/main/cpp/MetadataEntry.h b/test-app/runtime/src/main/cpp/MetadataEntry.h index 6e708526c..8035fed5c 100644 --- a/test-app/runtime/src/main/cpp/MetadataEntry.h +++ b/test-app/runtime/src/main/cpp/MetadataEntry.h @@ -50,8 +50,7 @@ namespace tns { clazz = other.clazz; parsedSig = other.parsedSig; mi = other.mi; - fi = other.fi; - sfi = other.sfi; + field = other.field; name = other.name; sig = other.sig; returnType = other.returnType; @@ -92,8 +91,8 @@ namespace tns { std::vector parsedSig; MethodInfo mi; - FieldInfo *fi; - StaticFieldInfo *sfi; + // A field's record in the value stream. + const uint8_t *field; std::string name; std::string sig; diff --git a/test-app/runtime/src/main/cpp/MetadataFieldInfo.h b/test-app/runtime/src/main/cpp/MetadataFieldInfo.h index 5951ad4cb..b9464aa2f 100644 --- a/test-app/runtime/src/main/cpp/MetadataFieldInfo.h +++ b/test-app/runtime/src/main/cpp/MetadataFieldInfo.h @@ -4,30 +4,16 @@ #include namespace tns { -struct __attribute__ ((__packed__)) FieldInfo { +/* + * The flags of a field record in the metadata's value stream. The record is the + * offset of the field's name, the id of its type and these flags. MetadataReader + * reads it, because how wide the first two are depends on the metadata. + */ +struct FieldInfo { static const uint8_t FINAL = 1; // On a static field: the id of its declaring type follows. A static field // without it is declared by the class whose value lists it. static const uint8_t HAS_DECLARING_TYPE = 1 << 1; - - FieldInfo() - : -nameOffset(0), nodeId(0), flags(0) { -} - -uint32_t nameOffset; -uint16_t nodeId; -uint8_t flags; -}; - -struct __attribute__ ((__packed__)) StaticFieldInfo: FieldInfo { - StaticFieldInfo() - : -FieldInfo(), declaringType(0) { -} - -// Part of the record only when HAS_DECLARING_TYPE is set. -uint16_t declaringType; }; } diff --git a/test-app/runtime/src/main/cpp/MetadataMethodInfo.cpp b/test-app/runtime/src/main/cpp/MetadataMethodInfo.cpp index ea9008c1e..01ed8ed51 100644 --- a/test-app/runtime/src/main/cpp/MetadataMethodInfo.cpp +++ b/test-app/runtime/src/main/cpp/MetadataMethodInfo.cpp @@ -1,41 +1,26 @@ #include "MetadataMethodInfo.h" #include "MetadataNode.h" -#include using namespace tns; -namespace { - // The value stream is packed, so nothing in it is aligned. - inline uint16_t ReadUInt16(const uint8_t *data) { - uint16_t value; - memcpy(&value, data, sizeof(value)); - return value; - } - - inline uint32_t ReadUInt32(const uint8_t *data) { - uint32_t value; - memcpy(&value, data, sizeof(value)); - return value; - } -} - std::string MethodInfo::GetName() { - string methodName = MetadataNode::getMetadataReader()->ReadName(ReadUInt32(m_record)); + auto m_reader = MetadataNode::getMetadataReader(); + string methodName = m_reader->ReadName(m_reader->ReadNameOffset(m_record)); return methodName; } uint8_t MethodInfo::CheckIsResolved() { - return m_record[sizeof(uint32_t)] & 1; + return m_record[MetadataNode::getMetadataReader()->NameBytes()] & 1; } const uint8_t *MethodInfo::Signature() { - // Read a byte at a time: the record can be the last thing in the stream, - // and a four-byte load would run past it. - const uint8_t *reference = m_record + sizeof(uint32_t); - uint32_t offset = (reference[0] | (reference[1] << 8) | (reference[2] << 16)) >> 1; + // The reference is the signature's offset in the signature region, shifted + // left by one to leave the low bit for the resolved flag. + auto m_reader = MetadataNode::getMetadataReader(); + uint32_t reference = m_reader->ReadSignatureReference(m_record + m_reader->NameBytes()); - return MetadataNode::getMetadataReader()->GetSignatureData() + offset; + return m_reader->GetSignatureData() + (reference >> 1); } uint16_t MethodInfo::GetSignatureLength() { @@ -51,7 +36,7 @@ std::string MethodInfo::GetSignature() { //use nodeId's to read the whole signat string signature = "("; string ret; for (int i = 0; i < signatureLength; i++) { - uint16_t nodeId = ReadUInt16(nodeIds + i * sizeof(uint16_t)); + uint32_t nodeId = m_reader->ReadId(nodeIds + i * m_reader->IdBytes()); string curArgTypeName = m_reader->ReadTypeName(nodeId); MetadataTreeNode* node = m_reader->GetNodeById(nodeId); @@ -86,9 +71,12 @@ std::string MethodInfo::GetSignature() { //use nodeId's to read the whole signat std::string MethodInfo::GetDeclaringType() { auto m_reader = MetadataNode::getMetadataReader(); - return m_reader->ReadTypeName(ReadUInt16(m_record + RECORD_SIZE)); + return m_reader->ReadTypeName( + m_reader->ReadId(m_record + m_reader->NameBytes() + m_reader->SignatureBytes())); } int MethodInfo::GetSizeOfReadMethodInfo() { - return RECORD_SIZE + (isStatic ? sizeof(uint16_t) : 0); + auto m_reader = MetadataNode::getMetadataReader(); + + return m_reader->NameBytes() + m_reader->SignatureBytes() + (isStatic ? m_reader->IdBytes() : 0); } diff --git a/test-app/runtime/src/main/cpp/MetadataMethodInfo.h b/test-app/runtime/src/main/cpp/MetadataMethodInfo.h index fc0b299cb..11e20a772 100644 --- a/test-app/runtime/src/main/cpp/MetadataMethodInfo.h +++ b/test-app/runtime/src/main/cpp/MetadataMethodInfo.h @@ -11,15 +11,14 @@ using namespace std; namespace tns { /* * A view of one method record in the metadata's value stream. The record is - * the offset of the method's name (4 bytes) and a reference to its signature - * (3 bytes); a static method or an extension function is followed by the id - * of its declaring type. Nothing is copied out of the record, and the - * signature itself is only read when the method is first resolved. + * the offset of the method's name and a reference to its signature; a static + * method or an extension function is followed by the id of its declaring + * type. MetadataReader knows how wide each of those is. Nothing is copied + * out of the record, and the signature itself is only read when the method + * is first resolved. */ class MethodInfo { public: - static const int RECORD_SIZE = sizeof(uint32_t) + 3; - MethodInfo(uint8_t *record) : isStatic(false), m_record(record) { } diff --git a/test-app/runtime/src/main/cpp/MetadataNode.cpp b/test-app/runtime/src/main/cpp/MetadataNode.cpp index 9d0675438..41e0e797e 100644 --- a/test-app/runtime/src/main/cpp/MetadataNode.cpp +++ b/test-app/runtime/src/main/cpp/MetadataNode.cpp @@ -820,8 +820,7 @@ void MetadataNode::SetInstanceMethodsFromStaticMetadata( SET_PROFILER_FRAME(); uint8_t sections = s_metadataReader.GetClassSections(treeNode); - curPtr = s_metadataReader.GetValueData() + treeNode->offsetValue - + MetadataReader::CLASS_VALUE_SECTIONS_OFFSET + sizeof(sections); + curPtr = s_metadataReader.GetClassMembers(treeNode); auto curType = s_metadataReader.ReadTypeName(treeNode); @@ -921,7 +920,7 @@ void MetadataNode::SetInstanceMethodsFromStaticMetadata( auto instanceFieldCout = MetadataReader::ReadSectionCount( sections, MetadataReader::SECTION_INSTANCE_FIELDS, &curPtr); for (auto i = 0; i < instanceFieldCout; i++) { - auto entry = MetadataReader::ReadInstanceFieldEntry(&curPtr); + auto entry = s_metadataReader.ReadInstanceFieldEntry(&curPtr); auto fieldName = entry.getName(); auto fieldInfo = cache->Own(entry); fieldInfo->metadata.declaringType = curType; @@ -931,9 +930,8 @@ void MetadataNode::SetInstanceMethodsFromStaticMetadata( auto kotlinPropertiesCount = MetadataReader::ReadSectionCount( sections, MetadataReader::SECTION_PROPERTIES, &curPtr); for (int i = 0; i < kotlinPropertiesCount; ++i) { - uint32_t nameOffset = *reinterpret_cast(curPtr); - string propertyName = s_metadataReader.ReadName(nameOffset); - curPtr += sizeof(uint32_t); + string propertyName = s_metadataReader.ReadName(s_metadataReader.ReadNameOffset(curPtr)); + curPtr += s_metadataReader.NameBytes(); uint8_t accessors = *curPtr++; @@ -1090,11 +1088,11 @@ void MetadataNode::SetStaticMembers(Isolate* isolate, Local& ctorFunct MoveToDictionaryMode(isolate, context, ctorFunction); } for (auto i = 0; i < staticFieldCout; i++) { - auto entry = MetadataReader::ReadStaticFieldEntry(&curPtr); + auto entry = s_metadataReader.ReadStaticFieldEntry(&curPtr); // A static field that does not name its declaring type is this class's own. entry.treeNode = treeNode; - auto fieldName = NewMemberName(isolate, s_metadataReader.ReadNameView(entry.sfi->nameOffset)); + auto fieldName = NewMemberName(isolate, s_metadataReader.ReadNameView(s_metadataReader.FieldNameOffset(entry.field))); auto fieldData = External::New(isolate, cache->Own(entry), v8::kExternalPointerTypeTagDefault); if (manyFields) { if (entry.getIsFinal()) { @@ -2267,14 +2265,6 @@ void MetadataNode::BuildMetadata(const string& filesPath) { } auto nodes = MapMetadataFile(baseDir, "treeNodeStream.dat"); - if ((nodes.length % sizeof(MetadataTreeNodeRawData)) != 0) { - stringstream ss; - ss << "Metadata file " << baseDir << "/treeNodeStream.dat is " << nodes.length - << " bytes, which is not a whole number of " - << sizeof(MetadataTreeNodeRawData) - << "-byte records. The metadata is truncated or corrupt."; - throw NativeScriptException(ss.str()); - } auto names = MapMetadataFile(baseDir, "treeStringsStream.dat"); auto values = MapMetadataFile(baseDir, "treeValueStream.dat"); @@ -2499,10 +2489,6 @@ std::string MetadataNode::GetJniClassName(MetadataEntry& entry) { return fullClassName; } -MetadataReader* MetadataNode::getMetadataReader() { - return &MetadataNode::s_metadataReader; -} - string MetadataNode::TNS_PREFIX = "com/tns/gen/"; MetadataReader MetadataNode::s_metadataReader; robin_hood::unordered_map MetadataNode::s_name2NodeCache; diff --git a/test-app/runtime/src/main/cpp/MetadataNode.h b/test-app/runtime/src/main/cpp/MetadataNode.h index 4a930a11a..6437b9932 100644 --- a/test-app/runtime/src/main/cpp/MetadataNode.h +++ b/test-app/runtime/src/main/cpp/MetadataNode.h @@ -75,7 +75,9 @@ class MetadataNode { static bool TryGetPackageName(v8::Isolate* isolate, const v8::Local& value, std::string& out); - static MetadataReader* getMetadataReader(); + static MetadataReader* getMetadataReader() { + return &s_metadataReader; + } private: struct MethodCallbackData; diff --git a/test-app/runtime/src/main/cpp/MetadataReader.cpp b/test-app/runtime/src/main/cpp/MetadataReader.cpp index c35658df0..bd6ae10c3 100644 --- a/test-app/runtime/src/main/cpp/MetadataReader.cpp +++ b/test-app/runtime/src/main/cpp/MetadataReader.cpp @@ -13,6 +13,8 @@ MetadataReader::MetadataReader() : m_root(nullptr), m_nodesLength(0), m_nameLeng m_valueLength(0), m_nodeData(nullptr), m_nameData(nullptr), m_valueData(nullptr), m_signatureData(nullptr), + m_idBytes(0), m_nameBytes(0), m_signatureBytes(0), + m_idMask(0), m_nameMask(0), m_signatureMask(0), m_getTypeMetadataCallback(nullptr) {} void MetadataReader::Initialize(uint32_t nodesLength, uint8_t *nodeData, uint32_t nameLength, @@ -21,12 +23,22 @@ void MetadataReader::Initialize(uint32_t nodesLength, uint8_t *nodeData, uint32_ StateLock lock(m_stateMutex); uint32_t format = 0; + uint32_t idBytes = 0; + uint32_t nameBytes = 0; + uint32_t signatureBytes = 0; uint32_t signaturesLength = 0; if (valueLength >= VALUE_HEADER_SIZE) { memcpy(&format, valueData, sizeof(format)); - memcpy(&signaturesLength, valueData + sizeof(format), sizeof(signaturesLength)); + idBytes = valueData[4]; + nameBytes = valueData[5]; + signatureBytes = valueData[6]; + memcpy(&signaturesLength, valueData + 2 * sizeof(uint32_t), sizeof(signaturesLength)); } - if (format != FORMAT || signaturesLength > valueLength - VALUE_HEADER_SIZE) { + auto isFieldWidth = [](uint32_t bytes) { return bytes >= 1 && bytes <= sizeof(uint32_t); }; + if (format != FORMAT + || !isFieldWidth(idBytes) || !isFieldWidth(nameBytes) || !isFieldWidth(signatureBytes) + || signaturesLength > valueLength - VALUE_HEADER_SIZE + || nodesLength % (2 * idBytes + nameBytes + sizeof(uint32_t)) != 0) { stringstream ss; ss << "The app's metadata is not in the format this runtime reads (found 0x" << hex << format << ", expected 0x" << FORMAT @@ -36,6 +48,15 @@ void MetadataReader::Initialize(uint32_t nodesLength, uint8_t *nodeData, uint32_ throw NativeScriptException(ss.str()); } + auto maskFor = [](uint32_t bytes) { + return bytes == sizeof(uint32_t) ? UINT32_MAX : (1u << (8 * bytes)) - 1; + }; + m_idBytes = idBytes; + m_nameBytes = nameBytes; + m_signatureBytes = signatureBytes; + m_idMask = maskFor(idBytes); + m_nameMask = maskFor(nameBytes); + m_signatureMask = maskFor(signatureBytes); m_signatureData = valueData + VALUE_HEADER_SIZE; m_nodesLength = nodesLength; @@ -109,59 +130,56 @@ void MetadataReader::StateMutex::Reacquire(unsigned depth) { //} MetadataTreeNode* MetadataReader::BuildTree() { - MetadataTreeNodeRawData* rootNodeData = reinterpret_cast(m_nodeData); - - MetadataTreeNodeRawData* curNodeData = rootNodeData; - - int len = m_nodesLength / sizeof(MetadataTreeNodeRawData); + // A node record: the ids of the node's first child and of its next sibling, + // either of which is the node's own id when there is none, then the offset + // of its name and the four-byte offset of its value. + const uint32_t recordSize = 2 * m_idBytes + m_nameBytes + sizeof(uint32_t); + const uint32_t len = m_nodesLength / recordSize; + + auto createNode = [&](uint32_t id) { + const uint8_t* record = m_nodeData + id * recordSize + 2 * m_idBytes; + auto node = new MetadataTreeNode; + node->name = ReadName(ReadNameOffset(record)); + memcpy(&node->offsetValue, record + m_nameBytes, sizeof(node->offsetValue)); + return node; + }; MetadataTreeNode* emptyNode = nullptr; m_v.assign(len, emptyNode); - for (int i = 0; i < len; i++) { + for (uint32_t i = 0; i < len; i++) { MetadataTreeNode* node = m_v[i]; if (nullptr == node) { - node = new MetadataTreeNode; - node->name = ReadName(curNodeData->offsetName); - node->offsetValue = curNodeData->offsetValue; + node = createNode(i); m_v[i] = node; } - uint16_t curNodeDataId = curNodeData - rootNodeData; + uint32_t childId = ReadId(m_nodeData + i * recordSize); - if (curNodeDataId != curNodeData->firstChildId) { + if (childId != i) { node->children = new vector; - MetadataTreeNodeRawData* childNodeData = rootNodeData + curNodeData->firstChildId; while (true) { - - uint16_t childNodeDataId = childNodeData - rootNodeData; - - MetadataTreeNode* childNode; // node (and its next siblings) already visited, so we don't need to visit it again - if (m_v[childNodeDataId] != emptyNode) { - childNode = m_v[childNodeDataId]; - __android_log_print(ANDROID_LOG_ERROR, "TNS.error", "Consistency error in metadata. A child should never have been visited before its parent. Parent: %s Child: %s. Child metadata id: %u", node->name.c_str(), childNode->name.c_str(), childNodeDataId); + if (childId >= len || m_v[childId] != emptyNode) { + __android_log_print(ANDROID_LOG_ERROR, "TNS.error", "Consistency error in metadata. A child should never have been visited before its parent. Parent: %s. Child metadata id: %u", node->name.c_str(), childId); break; - } else { - childNode = new MetadataTreeNode; - childNode->name = ReadName(childNodeData->offsetName); - childNode->offsetValue = childNodeData->offsetValue; } + + MetadataTreeNode* childNode = createNode(childId); childNode->parent = node; node->children->push_back(childNode); - m_v[childNodeDataId] = childNode; + m_v[childId] = childNode; - if (childNodeDataId == childNodeData->nextSiblingId) { + uint32_t siblingId = ReadId(m_nodeData + childId * recordSize + m_idBytes); + if (siblingId == childId) { break; } - childNodeData = rootNodeData + childNodeData->nextSiblingId; + childId = siblingId; } } - - curNodeData++; } return GetNodeById(0); @@ -496,8 +514,7 @@ MetadataTreeNode *MetadataReader::GetBaseClassNode(MetadataTreeNode *treeNode) { if (treeNode != nullptr && treeNode->metadata != nullptr) { baseClassNode = treeNode->runtimeBaseClass; } else if (treeNode != nullptr) { - uint16_t baseClassNodeId = *reinterpret_cast(m_valueData + - treeNode->offsetValue + 1); + uint32_t baseClassNodeId = ReadId(m_valueData + treeNode->offsetValue + sizeof(uint8_t)); // Null for an id the metadata got wrong, which GetNodeById logs; every // caller already treats a missing base class as "no base class". diff --git a/test-app/runtime/src/main/cpp/MetadataReader.h b/test-app/runtime/src/main/cpp/MetadataReader.h index dc404bdbb..0644fd6cc 100644 --- a/test-app/runtime/src/main/cpp/MetadataReader.h +++ b/test-app/runtime/src/main/cpp/MetadataReader.h @@ -24,20 +24,24 @@ namespace tns { * Writer.java produces and has to be changed together with: * * format 4 bytes, FORMAT + * widths the bytes a node id, an offset into the names and a + * signature reference each take (1 to 4), and a + * reserved byte. The generator picks the fewest that + * hold this metadata's largest value of each. * signatures length 4 bytes * signatures each distinct method signature once: a byte holding * the number of type ids less one, then the ids, the * return type first * values one per primitive type and per class or interface, * at the offset its tree node gives + * padding 4 bytes, so the last field can be read at full width * - * A class value is its type (1 byte), its base class id (2 bytes) and a - * byte with a bit for each member section it has. Those sections follow - * in the order of the bits, each a 2-byte count and its entries. + * A class value is its type (1 byte), its base class id and a byte with a + * bit for each member section it has. Those sections follow in the order + * of the bits, each a 2-byte count and its entries. */ - static constexpr uint32_t FORMAT = 'N' | ('S' << 8) | ('M' << 16) | (2 << 24); - static constexpr uint32_t VALUE_HEADER_SIZE = 2 * sizeof(uint32_t); - static constexpr uint32_t CLASS_VALUE_SECTIONS_OFFSET = sizeof(uint8_t) + sizeof(uint16_t); + static constexpr uint32_t FORMAT = 'N' | ('S' << 8) | ('M' << 16) | (3 << 24); + static constexpr uint32_t VALUE_HEADER_SIZE = 3 * sizeof(uint32_t); static constexpr uint8_t SECTION_EXTENSION_FUNCTIONS = 1; static constexpr uint8_t SECTION_INSTANCE_METHODS = 1 << 1; @@ -59,26 +63,80 @@ namespace tns { uint8_t *nameData, uint32_t valueLength, uint8_t *valueData, GetTypeMetadataCallback getTypeMetadataCallack); - inline static MetadataEntry ReadInstanceFieldEntry(uint8_t **data) { + /* + * A field of the node or value stream that is 1 to 4 bytes wide. It always + * loads four bytes and masks, so there is no branch on the width: the + * value stream is padded for its last field, and a node record ends in a + * full four-byte one. + */ + inline static uint32_t ReadField(const uint8_t *data, uint32_t mask) { + uint32_t value; + memcpy(&value, data, sizeof(value)); + return value & mask; + } + + inline uint32_t ReadId(const uint8_t *data) const { + return ReadField(data, m_idMask); + } + + inline uint32_t ReadNameOffset(const uint8_t *data) const { + return ReadField(data, m_nameMask); + } + + inline uint32_t ReadSignatureReference(const uint8_t *data) const { + return ReadField(data, m_signatureMask); + } + + inline uint32_t IdBytes() const { + return m_idBytes; + } + + inline uint32_t NameBytes() const { + return m_nameBytes; + } + + inline uint32_t SignatureBytes() const { + return m_signatureBytes; + } + + // A field record: its name, the id of its type, its flags (see FieldInfo) + // and, on a static field flagged for it, the id of its declaring type. + inline uint32_t FieldNameOffset(const uint8_t *field) const { + return ReadNameOffset(field); + } + + inline uint32_t FieldTypeId(const uint8_t *field) const { + return ReadId(field + m_nameBytes); + } + + inline uint8_t FieldFlags(const uint8_t *field) const { + return field[m_nameBytes + m_idBytes]; + } + + inline uint32_t FieldDeclaringTypeId(const uint8_t *field) const { + return ReadId(field + m_nameBytes + m_idBytes + sizeof(uint8_t)); + } + + inline MetadataEntry ReadInstanceFieldEntry(uint8_t **data) { MetadataEntry entry(nullptr, NodeType::Field); - entry.fi = *reinterpret_cast(data); + entry.field = *data; entry.isStatic = false; entry.isTypeMember = false; - *data += sizeof(FieldInfo); + *data += m_nameBytes + m_idBytes + sizeof(uint8_t); return entry; } - inline static MetadataEntry ReadStaticFieldEntry(uint8_t **data) { + inline MetadataEntry ReadStaticFieldEntry(uint8_t **data) { MetadataEntry entry(nullptr, NodeType::StaticField); - entry.sfi = *reinterpret_cast(data); + entry.field = *data; entry.isStatic = true; entry.isTypeMember = false; - *data += sizeof(FieldInfo); - if (entry.sfi->flags & FieldInfo::HAS_DECLARING_TYPE) { - *data += sizeof(uint16_t); + *data += m_nameBytes + m_idBytes + sizeof(uint8_t); + if (FieldFlags(entry.field) & FieldInfo::HAS_DECLARING_TYPE) { + *data += m_idBytes; } return entry; @@ -150,7 +208,13 @@ namespace tns { } inline uint8_t GetClassSections(MetadataTreeNode *treeNode) const { - return m_valueData[treeNode->offsetValue + CLASS_VALUE_SECTIONS_OFFSET]; + return m_valueData[treeNode->offsetValue + sizeof(uint8_t) + m_idBytes]; + } + + // Where a class value's member sections start: after its type, its base + // class id and the byte saying which sections it has. + inline uint8_t *GetClassMembers(MetadataTreeNode *treeNode) const { + return m_valueData + treeNode->offsetValue + sizeof(uint8_t) + m_idBytes + sizeof(uint8_t); } // The number of entries in one member section of a class value, leaving @@ -348,11 +412,10 @@ namespace tns { // stream's header, where no value starts. static const uint32_t RUNTIME_TYPE_VALUE_OFFSET = 1; - // The static metadata addresses its nodes with 16-bit ids. Nodes created at - // run time are numbered from here in every app, so a path that narrows one - // of their ids fails everywhere rather than only once the static tree is - // nearly full. - static const uint32_t RUNTIME_NODE_ID_BASE = 1 << 16; + // The static metadata numbers its nodes from 0, and the generator stops + // before this many. Nodes created at run time are numbered from here, which + // still leaves an array's marker, ARRAY_OFFSET plus an id, within 32 bits. + static const uint32_t RUNTIME_NODE_ID_BASE = 1 << 30; MetadataTreeNode *BuildTree(); @@ -379,6 +442,12 @@ namespace tns { uint8_t *m_nameData; uint8_t *m_valueData; const uint8_t *m_signatureData; + uint32_t m_idBytes; + uint32_t m_nameBytes; + uint32_t m_signatureBytes; + uint32_t m_idMask; + uint32_t m_nameMask; + uint32_t m_signatureMask; std::vector m_v; // Nodes created at run time, in creation order and by parent. They stay out // of MetadataTreeNode::children so the static tree never changes once diff --git a/test-app/runtime/src/main/cpp/MetadataTreeNode.h b/test-app/runtime/src/main/cpp/MetadataTreeNode.h index d32937d8f..a28275146 100644 --- a/test-app/runtime/src/main/cpp/MetadataTreeNode.h +++ b/test-app/runtime/src/main/cpp/MetadataTreeNode.h @@ -49,12 +49,6 @@ struct MetadataTreeNode { static const uint8_t INVALID_TYPE = 0xFF; }; -struct MetadataTreeNodeRawData { - uint16_t firstChildId; - uint16_t nextSiblingId; - uint32_t offsetName; - uint32_t offsetValue; -}; } #endif /* TREENODE_H_ */