Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update RegressionTest to assume UTF-8. #1006

Merged
merged 7 commits into from
Dec 5, 2024
Merged

Update RegressionTest to assume UTF-8. #1006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
15d0330
Update RegressionTest to assume UTF-8.
Nov 1, 2024
110f82b
Adds a requiresEncoding(Reader) method.
Nov 1, 2024
5201952
Added a BufferedReader.
Nov 1, 2024
8d24863
Minor type cleanup.
Nov 4, 2024
f455ee8
Switched to codepoints instead of characters.
Nov 4, 2024
4b58cf7
Merge branch 'main' into UpdateRegressionTestToUtf8
Nov 25, 2024
6049e62
Changes based on PR comments.
Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/main/java/emissary/core/IBaseDataObjectXmlCodecs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,11 +535,11 @@ public void encode(final List<SeekableByteChannelFactory> values, final Element
private static Element protectedElementHash(final String name, final byte[] bytes) {
final Element element = new Element(name);

if (ByteUtil.hasNonPrintableValues(bytes)) {
if (ByteUtil.containsNonIndexableBytes(bytes)) {
element.setAttribute(ENCODING_ATTRIBUTE_NAME, SHA256);
element.addContent(ByteUtil.sha256Bytes(bytes));
} else {
element.addContent(new String(bytes, StandardCharsets.ISO_8859_1));
element.addContent(new String(bytes, StandardCharsets.UTF_8));
}

return element;
Expand Down Expand Up @@ -777,15 +777,15 @@ public static Element protectedElement(final String name, final String string) {
public static Element protectedElementBase64(final String name, final byte[] bytes) {
final Element element = new Element(name);

if (ByteUtil.hasNonPrintableValues(bytes)) {
if (ByteUtil.containsNonIndexableBytes(bytes)) {
String base64String = BASE64_NEW_LINE_STRING +
BASE64_ENCODER.encodeToString(bytes) +
BASE64_NEW_LINE_STRING;

element.setAttribute(ENCODING_ATTRIBUTE_NAME, BASE64);
element.addContent(base64String);
} else {
element.addContent(new String(bytes, StandardCharsets.ISO_8859_1));
element.addContent(new String(bytes, StandardCharsets.UTF_8));
}

return element;
Expand All @@ -803,11 +803,11 @@ public static Element protectedElementBase64(final String name, final byte[] byt
public static Element protectedElementSha256(final String name, final byte[] bytes) {
final Element element = new Element(name);

if (ByteUtil.hasNonPrintableValues(bytes)) {
if (ByteUtil.containsNonIndexableBytes(bytes)) {
element.setAttribute(IBaseDataObjectXmlCodecs.ENCODING_ATTRIBUTE_NAME, IBaseDataObjectXmlCodecs.SHA256);
element.addContent(ByteUtil.sha256Bytes(bytes));
} else {
element.addContent(new String(bytes, StandardCharsets.ISO_8859_1));
element.addContent(new String(bytes, StandardCharsets.UTF_8));
}

return element;
Expand Down
28 changes: 14 additions & 14 deletions src/test/java/emissary/core/IBaseDataObjectXmlHelperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ private static void setAllFieldsPrintable(final IBaseDataObject ibdo, final byte
ibdo.setFilename("Filename");
ibdo.setFileType("FileType");
ibdo.setFontEncoding("FontEncoding");
ibdo.setFooter("Footer".getBytes(StandardCharsets.ISO_8859_1));
ibdo.setHeader("Header".getBytes(StandardCharsets.ISO_8859_1));
ibdo.setFooter("Footer".getBytes(StandardCharsets.UTF_8));
ibdo.setHeader("Header".getBytes(StandardCharsets.UTF_8));
ibdo.setHeaderEncoding("HeaderEncoding");
ibdo.setId("Id");
ibdo.setNumChildren(9);
Expand All @@ -84,8 +84,8 @@ private static void setAllFieldsPrintable(final IBaseDataObject ibdo, final byte
ibdo.putParameter("Parameter1Key", "Parameter1Value");
ibdo.putParameter("Parameter2Key", Arrays.asList("Parameter2Value1", "Parameter2Value2"));
ibdo.putParameter("Parameter3Key", Arrays.asList(10L, 20L));
ibdo.addAlternateView("AlternateView1Key", "AlternateView1Value".getBytes(StandardCharsets.ISO_8859_1));
ibdo.addAlternateView("AlternateView11Key", "AlternateView11Value".getBytes(StandardCharsets.ISO_8859_1));
ibdo.addAlternateView("AlternateView1Key", "AlternateView1Value".getBytes(StandardCharsets.UTF_8));
ibdo.addAlternateView("AlternateView11Key", "AlternateView11Value".getBytes(StandardCharsets.UTF_8));
}

private static void setAllFieldsNonPrintable(final IBaseDataObject ibdo, final byte[] bytes) {
Expand All @@ -99,8 +99,8 @@ private static void setAllFieldsNonPrintable(final IBaseDataObject ibdo, final b
ibdo.setFilename("\001Filename");
ibdo.setFileType("\001FileType");
ibdo.setFontEncoding("\001FontEncoding");
ibdo.setFooter("\001Footer".getBytes(StandardCharsets.ISO_8859_1));
ibdo.setHeader("\001Header".getBytes(StandardCharsets.ISO_8859_1));
ibdo.setFooter("\001Footer".getBytes(StandardCharsets.UTF_8));
ibdo.setHeader("\001Header".getBytes(StandardCharsets.UTF_8));
ibdo.setHeaderEncoding("\001HeaderEncoding");
ibdo.setId("\001Id");
ibdo.setNumChildren(9);
Expand All @@ -114,9 +114,9 @@ private static void setAllFieldsNonPrintable(final IBaseDataObject ibdo, final b
ibdo.putParameter("\020Parameter1Key", "\020Parameter1Value");
ibdo.putParameter("\020Parameter2Key", "\020Parameter2Value");
ibdo.addAlternateView("\200AlternateView1Key",
"\200AlternateView1Value".getBytes(StandardCharsets.ISO_8859_1));
"\200AlternateView1Value".getBytes(StandardCharsets.UTF_8));
ibdo.addAlternateView("\200AlternateView11Key",
"\200AlternateView11Value".getBytes(StandardCharsets.ISO_8859_1));
"\200AlternateView11Value".getBytes(StandardCharsets.UTF_8));
}

@Test
Expand All @@ -125,7 +125,7 @@ void testParentIbdoAllFieldsChanged() throws Exception {
final IBaseDataObject expectedIbdo = new BaseDataObject();
final List<IBaseDataObject> expectedChildren = new ArrayList<>();
final List<IBaseDataObject> actualChildren = new ArrayList<>();
final byte[] bytes = "Data".getBytes(StandardCharsets.ISO_8859_1);
final byte[] bytes = "Data".getBytes(StandardCharsets.UTF_8);

setAllFieldsPrintable(expectedIbdo, bytes);

Expand All @@ -150,7 +150,7 @@ void testBase64Conversion() throws Exception {
final IBaseDataObject expectedIbdo = new BaseDataObject();
final List<IBaseDataObject> expectedChildren = new ArrayList<>();
final List<IBaseDataObject> actualChildren = new ArrayList<>();
final byte[] bytes = "\001Data".getBytes(StandardCharsets.ISO_8859_1);
final byte[] bytes = "\001Data".getBytes(StandardCharsets.UTF_8);

setAllFieldsNonPrintable(expectedIbdo, bytes);

Expand All @@ -164,10 +164,10 @@ void testBase64Conversion() throws Exception {
final IBaseDataObject sha256ActualIbdo = ibdoFromXmlFromIbdo(expectedIbdo, expectedChildren, initialIbdo,
actualChildren, SHA256_ELEMENT_ENCODERS);

expectedIbdo.setData(ByteUtil.sha256Bytes(bytes).getBytes(StandardCharsets.ISO_8859_1));
expectedIbdo.setData(ByteUtil.sha256Bytes(bytes).getBytes(StandardCharsets.UTF_8));

for (Entry<String, byte[]> entry : new TreeMap<>(expectedIbdo.getAlternateViews()).entrySet()) {
expectedIbdo.addAlternateView(entry.getKey(), ByteUtil.sha256Bytes(entry.getValue()).getBytes(StandardCharsets.ISO_8859_1));
expectedIbdo.addAlternateView(entry.getKey(), ByteUtil.sha256Bytes(entry.getValue()).getBytes(StandardCharsets.UTF_8));
}

final String sha256Diff = PlaceComparisonHelper.checkDifferences(expectedIbdo, sha256ActualIbdo, expectedChildren,
Expand All @@ -180,7 +180,7 @@ void testBase64Conversion() throws Exception {
void testLengthAttributeDefault() throws Exception {
final IBaseDataObject ibdo = new BaseDataObject();
final List<IBaseDataObject> children = new ArrayList<>();
final byte[] bytes = "Data".getBytes(StandardCharsets.ISO_8859_1);
final byte[] bytes = "Data".getBytes(StandardCharsets.UTF_8);

setAllFieldsPrintable(ibdo, bytes);

Expand All @@ -201,7 +201,7 @@ void testLengthAttributeDefault() throws Exception {
void testLengthAttributeHash() throws Exception {
final IBaseDataObject ibdo = new BaseDataObject();
final List<IBaseDataObject> children = new ArrayList<>();
final byte[] bytes = "Data".getBytes(StandardCharsets.ISO_8859_1);
final byte[] bytes = "Data".getBytes(StandardCharsets.UTF_8);

setAllFieldsNonPrintable(ibdo, bytes);

Expand Down
4 changes: 2 additions & 2 deletions src/test/java/emissary/test/core/junit5/RegressionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ protected void checkAnswersPreHook(final Document answers, final IBaseDataObject
// touch up alternate views to match how their bytes would have encoded into the answer file
for (Entry<String, byte[]> entry : new TreeMap<>(payload.getAlternateViews()).entrySet()) {
Optional<String> viewSha256 = hashBytesIfNonPrintable(entry.getValue());
viewSha256.ifPresent(s -> payload.addAlternateView(entry.getKey(), s.getBytes(StandardCharsets.ISO_8859_1)));
viewSha256.ifPresent(s -> payload.addAlternateView(entry.getKey(), s.getBytes(StandardCharsets.UTF_8)));
}

// touch up primary view if necessary
Expand Down Expand Up @@ -233,7 +233,7 @@ protected void checkAnswersPreHookLogEvents(List<SimplifiedLogEvent> simplifiedL
* @return a value optionally containing the generated hash
*/
protected Optional<String> hashBytesIfNonPrintable(byte[] bytes) {
if (ArrayUtils.isNotEmpty(bytes) && ByteUtil.hasNonPrintableValues(bytes)) {
if (ArrayUtils.isNotEmpty(bytes) && ByteUtil.containsNonIndexableBytes(bytes)) {
return Optional.ofNullable(ByteUtil.sha256Bytes(bytes));
}

Expand Down