Skip to content

[GR-80729] Add a PE-friendly JSON memo table #1234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graalvmbot merged 2 commits into from
Oct 20, 2026
Merged

[GR-80729] Add a PE-friendly JSON memo table #1234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d13a597
Add a PE-friendly JSON memo table
msimacek Oct 19, 2026
ae1990e
Use better hash spread function in json memo map
msimacek Oct 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 11 additions & 16 deletions ...e.graal.python/src/com/oracle/graal/python/builtins/modules/json/JSONScannerBuiltins.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
import static com.oracle.graal.python.util.PythonUtils.tsLiteral;

import java.util.HashMap;
import java.util.List;

import com.oracle.graal.python.PythonLanguage;
Expand All @@ -26,6 +25,7 @@
import com.oracle.graal.python.builtins.CoreFunctions;
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
import com.oracle.graal.python.builtins.PythonBuiltins;
import com.oracle.graal.python.builtins.modules.json.PJSONScanner.KeyMemo;
import com.oracle.graal.python.builtins.objects.PNone;
import com.oracle.graal.python.builtins.objects.common.EconomicMapStorage;
import com.oracle.graal.python.builtins.objects.common.HashingStorage;
Expand Down Expand Up @@ -174,6 +174,7 @@ protected PTuple call(VirtualFrame frame, PJSONScanner self, TruffleString strin
@Cached PyLongFromUnicodeObject pyLongFromUnicodeObject,
@Cached TruffleString.MaterializeNode materializeNode,
@Cached TruffleString.HashCodeNode hashCodeNode,
@Cached TruffleString.EqualNode equalNode,
@Cached TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode,
@Cached TruffleString.RegionEqualByteIndexNode regionEqualByteIndexNode,
@Cached TruffleString.SubstringByteIndexNode substringByteIndexNode,
Expand Down Expand Up @@ -210,6 +211,7 @@ protected PTuple call(VirtualFrame frame, PJSONScanner self, TruffleString strin
hashingStorageSetItem,
pyLongFromUnicodeObject,
hashCodeNode,
equalNode,
codePointAtIndexNode,
regionEqualByteIndexNode,
substringByteIndexNode,
Expand All @@ -232,6 +234,7 @@ protected PTuple call(VirtualFrame frame, PJSONScanner self, TruffleString strin
hashingStorageSetItem,
pyLongFromUnicodeObject,
hashCodeNode,
equalNode,
codePointAtIndexNode,
regionEqualByteIndexNode,
substringByteIndexNode,
Expand Down Expand Up @@ -374,7 +377,7 @@ private enum ScannerState {

@InliningCutoff
private Object scanOnceUnicodeCutoff(VirtualFrame frame, BoundaryCallData boundaryCallData, Node inliningTarget, TruffleString string, int idx, IntRef nextIdx,
HashMap<TruffleString, TruffleString> memo,
KeyMemo memo,
boolean strict,
Object objectHook,
Object objectPairsHook,
Expand All @@ -385,6 +388,7 @@ private Object scanOnceUnicodeCutoff(VirtualFrame frame, BoundaryCallData bounda
HashingStorageSetItem hashingStorageSetItem,
PyLongFromUnicodeObject pyLongFromUnicodeObject,
TruffleString.HashCodeNode hashCodeNode,
TruffleString.EqualNode equalNode,
TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode,
TruffleString.RegionEqualByteIndexNode regionEqualByteIndexNode,
TruffleString.SubstringByteIndexNode substringByteIndexNode,
Expand All @@ -401,7 +405,7 @@ private Object scanOnceUnicodeCutoff(VirtualFrame frame, BoundaryCallData bounda
parseInt,
parseConstant,
errorProfile,
hashingStorageSetItem, pyLongFromUnicodeObject, hashCodeNode,
hashingStorageSetItem, pyLongFromUnicodeObject, hashCodeNode, equalNode,
codePointAtIndexNode,
regionEqualByteIndexNode,
substringByteIndexNode,
Expand All @@ -414,7 +418,7 @@ private Object scanOnceUnicodeCutoff(VirtualFrame frame, BoundaryCallData bounda
}

private Object scanOnceUnicode(VirtualFrame frame, BoundaryCallData boundaryCallData, Node inliningTarget, TruffleString string, int idx, IntRef nextIdx,
HashMap<TruffleString, TruffleString> memo,
KeyMemo memo,
boolean strict,
Object objectHook,
Object objectPairsHook,
Expand All @@ -425,6 +429,7 @@ private Object scanOnceUnicode(VirtualFrame frame, BoundaryCallData boundaryCall
HashingStorageSetItem hashingStorageSetItem,
PyLongFromUnicodeObject pyLongFromUnicodeObject,
TruffleString.HashCodeNode hashCodeNode,
TruffleString.EqualNode equalNode,
TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode,
TruffleString.RegionEqualByteIndexNode regionEqualByteIndexNode,
TruffleString.SubstringByteIndexNode substringByteIndexNode,
Expand Down Expand Up @@ -551,13 +556,8 @@ private Object scanOnceUnicode(VirtualFrame frame, BoundaryCallData boundaryCall
substringByteIndexNode,
appendCodePointNode,
appendSubstringByteIndexNode, builderToStringNode);
/* force hash computation */
hashCodeNode.execute(newKey, TS_ENCODING);
TruffleString key = memoPutIfAbsent(memo, newKey);
if (key == null) {
key = newKey;
}
propertyKey = key;
int hash = hashCodeNode.execute(newKey, TS_ENCODING);
propertyKey = memo.intern(newKey, hash, equalNode);
idx = nextIdx.value;
/*
* skip whitespace between key and : delimiter, read :, skip whitespace
Expand Down Expand Up @@ -700,11 +700,6 @@ private Object scanOnceUnicode(VirtualFrame frame, BoundaryCallData boundaryCall
}
}

@TruffleBoundary
private static TruffleString memoPutIfAbsent(HashMap<TruffleString, TruffleString> memo, TruffleString newKey) {
return memo.putIfAbsent(newKey, newKey);
}

private static final TruffleString[] DOUBLE_CONSTANTS = {
tsLiteral("NaN"),
tsLiteral("Infinity"),
Expand Down
73 changes: 69 additions & 4 deletions ...m.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/json/PJSONScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* The Universal Permissive License (UPL), Version 1.0
Expand Down Expand Up @@ -40,16 +40,17 @@
*/
package com.oracle.graal.python.builtins.modules.json;

import java.util.HashMap;
import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING;

import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
import com.oracle.truffle.api.CompilerAsserts;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.object.Shape;
import com.oracle.truffle.api.strings.TruffleString;

public final class PJSONScanner extends PythonBuiltinObject {

final HashMap<TruffleString, TruffleString> memo;
final KeyMemo memo;
final boolean strict;
final Object objectHook;
final Object objectPairsHook;
Expand All @@ -60,12 +61,76 @@ public final class PJSONScanner extends PythonBuiltinObject {
public PJSONScanner(Object cls, Shape instanceShape, boolean strict, Object objectHook, Object objectPairsHook, Object parseFloat, Object parseInt, Object parseConstant) {
super(cls, instanceShape);
CompilerAsserts.neverPartOfCompilation();
this.memo = new HashMap<>();
this.memo = new KeyMemo();
this.strict = strict;
this.objectHook = objectHook;
this.objectPairsHook = objectPairsHook;
this.parseFloat = parseFloat;
this.parseInt = parseInt;
this.parseConstant = parseConstant;
}

/** Interns JSON property names without calling the uncached Java string equality methods. */
static final class KeyMemo {
private TruffleString[] keys = new TruffleString[16];
private int size;

TruffleString intern(TruffleString key, int hash, TruffleString.EqualNode equalNode) {
TruffleString[] table = keys;
int mask = table.length - 1;
int index = spread(hash) & mask;
TruffleString existing;
while ((existing = table[index]) != null) {
if (equalNode.execute(existing, key, TS_ENCODING)) {
return existing;
}
index = (index + 1) & mask;
}
if (size < table.length - (table.length >> 2)) {
table[index] = key;
size++;
} else {
growAndInsert(key, hash);
}
return key;
}

private static int spread(int hash) {
// MurmurHash3 finalizer: mix all bits before masking so similar property names do not
// cluster in adjacent slots during linear probing.
hash ^= hash >>> 16;
hash *= 0x85ebca6b;
hash ^= hash >>> 13;
hash *= 0xc2b2ae35;
return hash ^ (hash >>> 16);
}

@TruffleBoundary
private void growAndInsert(TruffleString key, int hash) {
TruffleString[] oldKeys = keys;
if (oldKeys.length == 1 << 30) {
throw new OutOfMemoryError();
}
TruffleString[] newKeys = new TruffleString[oldKeys.length << 1];
for (TruffleString existing : oldKeys) {
if (existing != null) {
// All entries were hashed by HashCodeNode before being inserted.
insert(newKeys, existing, existing.hashCode());
}
}
insert(newKeys, key, hash);
keys = newKeys;
size++;
}

private static void insert(TruffleString[] table, TruffleString key, int hash) {
int mask = table.length - 1;
int index = spread(hash) & mask;
while (table[index] != null) {
index = (index + 1) & mask;
}
table[index] = key;
}
}

}
Loading