From 60deebb18e9a3ae353638c6bbdf76842ff5c5930 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 17 Oct 2023 11:47:32 -0500 Subject: [PATCH 01/56] Make this a bit more robust * Hold array reference in local var while accessing * Use copyOfRange to realloc array --- core/src/main/java/org/jruby/RubyString.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyString.java b/core/src/main/java/org/jruby/RubyString.java index 02a954b31b8..ef45b4f58e0 100644 --- a/core/src/main/java/org/jruby/RubyString.java +++ b/core/src/main/java/org/jruby/RubyString.java @@ -744,7 +744,8 @@ public static EmptyByteListHolder getEmptyByteList(Encoding enc) { if (enc == null) enc = ASCIIEncoding.INSTANCE; int index = enc.getIndex(); EmptyByteListHolder bytes; - if (index < EMPTY_BYTELISTS.length && (bytes = EMPTY_BYTELISTS[index]) != null) { + EmptyByteListHolder[] emptyBytelists = EMPTY_BYTELISTS; + if (index < emptyBytelists.length && (bytes = emptyBytelists[index]) != null) { return bytes; } return prepareEmptyByteList(enc); @@ -753,12 +754,11 @@ public static EmptyByteListHolder getEmptyByteList(Encoding enc) { private static EmptyByteListHolder prepareEmptyByteList(Encoding enc) { if (enc == null) enc = ASCIIEncoding.INSTANCE; int index = enc.getIndex(); - if (index >= EMPTY_BYTELISTS.length) { - EmptyByteListHolder tmp[] = new EmptyByteListHolder[index + 4]; - System.arraycopy(EMPTY_BYTELISTS,0, tmp, 0, EMPTY_BYTELISTS.length); - EMPTY_BYTELISTS = tmp; + EmptyByteListHolder[] emptyBytelists = EMPTY_BYTELISTS; + if (index >= emptyBytelists.length) { + EMPTY_BYTELISTS = emptyBytelists = Arrays.copyOfRange(emptyBytelists, 0, index + 4); } - return EMPTY_BYTELISTS[index] = new EmptyByteListHolder(enc); + return emptyBytelists[index] = new EmptyByteListHolder(enc); } public static RubyString newEmptyString(Ruby runtime, RubyClass metaClass, Encoding enc) { From b9d655c5edb559601dbdfd8f18248519af7cd946 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Wed, 18 Oct 2023 10:06:36 -0500 Subject: [PATCH 02/56] Move string-related bootstrapping to StringBootstrap --- .../org/jruby/ir/targets/indy/Bootstrap.java | 123 --------------- .../ir/targets/indy/IndyValueCompiler.java | 11 +- .../ir/targets/indy/RegexpObjectSite.java | 2 +- .../ir/targets/indy/StringBootstrap.java | 142 ++++++++++++++++++ 4 files changed, 148 insertions(+), 130 deletions(-) create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/StringBootstrap.java diff --git a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java index cd3d6f09581..937a9d57416 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java @@ -31,7 +31,6 @@ import com.headius.invokebinder.SmartBinder; import com.headius.invokebinder.SmartHandle; import org.jcodings.Encoding; -import org.jcodings.EncodingDB; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyBasicObject; @@ -87,7 +86,6 @@ import org.jruby.util.ByteList; import org.jruby.util.CodegenUtils; import org.jruby.util.JavaNameMangler; -import org.jruby.util.StringSupport; import org.jruby.util.cli.Options; import org.jruby.util.log.Logger; import org.jruby.util.log.LoggerFactory; @@ -126,55 +124,8 @@ public class Bootstrap { public final static String BOOTSTRAP_INT_SIG = sig(CallSite.class, Lookup.class, String.class, MethodType.class, int.class); private static final Logger LOG = LoggerFactory.getLogger(Bootstrap.class); static final Lookup LOOKUP = MethodHandles.lookup(); - public static final Handle EMPTY_STRING_BOOTSTRAP = new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "emptyString", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class), - false); - public static final Handle BUFFER_STRING_BOOTSTRAP = new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "bufferString", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, int.class), - false); private static final String[] GENERIC_CALL_PERMUTE = {"context", "self", "arg.*"}; - public static CallSite string(Lookup lookup, String name, MethodType type, String value, String encodingName, int cr) { - return new ConstantCallSite(insertArguments(STRING_HANDLE, 1, bytelist(value, encodingName), cr)); - } - - private static final MethodHandle STRING_HANDLE = - Binder - .from(RubyString.class, ThreadContext.class, ByteList.class, int.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "string"); - - public static CallSite fstring(Lookup lookup, String name, MethodType type, String value, String encodingName, int cr, String file, int line) { - MutableCallSite site = new MutableCallSite(type); - - site.setTarget(insertArguments(FSTRING_HANDLE, 1, site, bytelist(value, encodingName), cr, file, line)); - - return site; - } - - private static final MethodHandle FSTRING_HANDLE = - Binder - .from(RubyString.class, ThreadContext.class, MutableCallSite.class, ByteList.class, int.class, String.class, int.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "frozenString"); - - public static CallSite emptyString(Lookup lookup, String name, MethodType type, String encodingName) { - RubyString.EmptyByteListHolder holder = RubyString.getEmptyByteList(encodingFromName(encodingName)); - return new ConstantCallSite(insertArguments(STRING_HANDLE, 1, holder.bytes, holder.cr)); - } - - public static CallSite bufferString(Lookup lookup, String name, MethodType type, String encodingName, int size) { - return new ConstantCallSite(insertArguments(BUFFERSTRING_HANDLE, 1, encodingFromName(encodingName), size, StringSupport.CR_7BIT)); - } - - private static final MethodHandle BUFFERSTRING_HANDLE = - Binder - .from(RubyString.class, ThreadContext.class, Encoding.class, int.class, int.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "bufferString"); public static Handle isNilBoot() { return new Handle( @@ -266,36 +217,6 @@ public static boolean isTruthy(IRubyObject obj, RubyNil nil, RubyBoolean.False f } } - public static CallSite bytelist(Lookup lookup, String name, MethodType type, String value, String encodingName) { - return new ConstantCallSite(constant(ByteList.class, bytelist(value, encodingName))); - } - - public static ByteList bytelist(String value, String encodingName) { - Encoding encoding = encodingFromName(encodingName); - - if (value.length() == 0) { - // special case empty string and don't create a new BL - return RubyString.getEmptyByteList(encoding).bytes; - } - - return new ByteList(RubyEncoding.encodeISO(value), encoding, false); - } - - public static ByteList bytelist(int size, String encodingName) { - Encoding encoding = encodingFromName(encodingName); - - return new ByteList(size, encoding); - } - - private static Encoding encodingFromName(String encodingName) { - Encoding encoding; - EncodingDB.Entry entry = EncodingDB.getEncodings().get(encodingName.getBytes()); - if (entry == null) entry = EncodingDB.getAliases().get(encodingName.getBytes()); - if (entry == null) throw new RuntimeException("could not find encoding: " + encodingName); - encoding = entry.getEncoding(); - return encoding; - } - public static final Handle CALLSITE = new Handle( Opcodes.H_INVOKESTATIC, p(Bootstrap.class), @@ -456,33 +377,6 @@ public static CallSite kwargsHash(Lookup lookup, String name, MethodType type) { .from(RubyHash.class, ThreadContext.class, RubyHash.class, IRubyObject[].class) .invokeStaticQuiet(LOOKUP, Bootstrap.class, "kwargsHash"); - public static Handle string() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "string", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, String.class, int.class), - false); - } - - public static Handle fstring() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "fstring", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, String.class, int.class, String.class, int.class), - false); - } - - public static Handle bytelist() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "bytelist", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, String.class), - false); - } - public static Handle array() { return new Handle( Opcodes.H_INVOKESTATIC, @@ -523,23 +417,6 @@ public static Handle global() { false); } - public static RubyString string(ThreadContext context, ByteList value, int cr) { - return RubyString.newStringShared(context.runtime, value, cr); - } - - public static RubyString bufferString(ThreadContext context, Encoding encoding, int size, int cr) { - return RubyString.newString(context.runtime, new ByteList(size, encoding), cr); - } - - public static RubyString frozenString(ThreadContext context, MutableCallSite site, ByteList value, int cr, String file, int line) { - RubyString frozen = IRRuntimeHelpers.newFrozenString(context, value, cr, file, line); - - // Permanently bind to the new frozen string - site.setTarget(dropArguments(constant(RubyString.class, frozen), 0, ThreadContext.class)); - - return frozen; - } - public static RubyArray array(ThreadContext context, IRubyObject[] ary) { assert ary.length > RubyArraySpecialized.MAX_PACKED_SIZE; // Bootstrap.array() only dispatches here if ^^ holds diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java index d630c9cd8c4..75f86caa99f 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java @@ -24,7 +24,6 @@ import org.jruby.util.CodegenUtils; import java.math.BigInteger; -import java.util.function.Consumer; import static org.jruby.util.CodegenUtils.ci; import static org.jruby.util.CodegenUtils.p; @@ -74,26 +73,26 @@ public void pushFloat(double d) { public void pushString(ByteList bl, int cr) { compiler.loadContext(); - compiler.adapter.invokedynamic("string", sig(RubyString.class, ThreadContext.class), Bootstrap.string(), RubyEncoding.decodeRaw(bl), bl.getEncoding().toString(), cr); + compiler.adapter.invokedynamic("string", sig(RubyString.class, ThreadContext.class), StringBootstrap.STRING_BOOTSTRAP, RubyEncoding.decodeRaw(bl), bl.getEncoding().toString(), cr); } public void pushFrozenString(ByteList bl, int cr, String file, int line) { compiler.loadContext(); - compiler.adapter.invokedynamic("frozen", sig(RubyString.class, ThreadContext.class), Bootstrap.fstring(), RubyEncoding.decodeRaw(bl), bl.getEncoding().toString(), cr, file, line); + compiler.adapter.invokedynamic("frozen", sig(RubyString.class, ThreadContext.class), StringBootstrap.FSTRING_BOOTSTRAP, RubyEncoding.decodeRaw(bl), bl.getEncoding().toString(), cr, file, line); } public void pushEmptyString(Encoding encoding) { compiler.loadContext(); - compiler.adapter.invokedynamic("emptyString", sig(RubyString.class, ThreadContext.class), Bootstrap.EMPTY_STRING_BOOTSTRAP, encoding.toString()); + compiler.adapter.invokedynamic("emptyString", sig(RubyString.class, ThreadContext.class), StringBootstrap.EMPTY_STRING_BOOTSTRAP, encoding.toString()); } public void pushBufferString(Encoding encoding, int size) { compiler.loadContext(); - compiler.adapter.invokedynamic("bufferString", sig(RubyString.class, ThreadContext.class), Bootstrap.BUFFER_STRING_BOOTSTRAP, encoding.toString(), size); + compiler.adapter.invokedynamic("bufferString", sig(RubyString.class, ThreadContext.class), StringBootstrap.BUFFER_STRING_BOOTSTRAP, encoding.toString(), size); } public void pushByteList(ByteList bl) { - compiler.adapter.invokedynamic("bytelist", sig(ByteList.class), Bootstrap.bytelist(), RubyEncoding.decodeRaw(bl), bl.getEncoding().toString()); + compiler.adapter.invokedynamic("bytelist", sig(ByteList.class), StringBootstrap.BYTELIST_BOOTSTRAP, RubyEncoding.decodeRaw(bl), bl.getEncoding().toString()); } public void pushRange(Runnable begin, Runnable end, boolean exclusive) { diff --git a/core/src/main/java/org/jruby/ir/targets/indy/RegexpObjectSite.java b/core/src/main/java/org/jruby/ir/targets/indy/RegexpObjectSite.java index 84aa08f1a67..076962558c1 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/RegexpObjectSite.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/RegexpObjectSite.java @@ -38,7 +38,7 @@ public RegexpObjectSite(MethodType type, ByteList pattern, int embeddedOptions) false); public static CallSite bootstrap(MethodHandles.Lookup lookup, String name, MethodType type, String value, String encodingName, int options) { - return new RegexpObjectSite(type, Bootstrap.bytelist(value, encodingName), options).bootstrap(lookup); + return new RegexpObjectSite(type, StringBootstrap.bytelist(value, encodingName), options).bootstrap(lookup); } // normal regexp diff --git a/core/src/main/java/org/jruby/ir/targets/indy/StringBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/StringBootstrap.java new file mode 100644 index 00000000000..b4f0c93ca4b --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/StringBootstrap.java @@ -0,0 +1,142 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jcodings.Encoding; +import org.jcodings.EncodingDB; +import org.jruby.RubyEncoding; +import org.jruby.RubyString; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.runtime.ThreadContext; +import org.jruby.util.ByteList; +import org.jruby.util.StringSupport; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; + +import static java.lang.invoke.MethodHandles.constant; +import static java.lang.invoke.MethodHandles.dropArguments; +import static java.lang.invoke.MethodHandles.insertArguments; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class StringBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + public static final Handle BYTELIST_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(StringBootstrap.class), + "bytelist", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, String.class), + false); + public static final Handle STRING_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(StringBootstrap.class), + "string", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, String.class, int.class), + false); + public static final Handle EMPTY_STRING_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(StringBootstrap.class), + "emptyString", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class), + false); + public static final Handle BUFFER_STRING_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(StringBootstrap.class), + "bufferString", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, int.class), + false); + public static final Handle FSTRING_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(StringBootstrap.class), + "fstring", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, String.class, int.class, String.class, int.class), + false); + + private static final MethodHandle STRING_HANDLE = + Binder + .from(RubyString.class, ThreadContext.class, ByteList.class, int.class) + .invokeStaticQuiet(LOOKUP, StringBootstrap.class, "string"); + private static final MethodHandle FSTRING_HANDLE = + Binder + .from(RubyString.class, ThreadContext.class, MutableCallSite.class, ByteList.class, int.class, String.class, int.class) + .invokeStaticQuiet(LOOKUP, StringBootstrap.class, "frozenString"); + private static final MethodHandle BUFFERSTRING_HANDLE = + Binder + .from(RubyString.class, ThreadContext.class, Encoding.class, int.class, int.class) + .invokeStaticQuiet(LOOKUP, StringBootstrap.class, "bufferString"); + + public static CallSite bytelist(MethodHandles.Lookup lookup, String name, MethodType type, String value, String encodingName) { + return new ConstantCallSite(constant(ByteList.class, bytelist(value, encodingName))); + } + + public static CallSite string(MethodHandles.Lookup lookup, String name, MethodType type, String value, String encodingName, int cr) { + return new ConstantCallSite(insertArguments(STRING_HANDLE, 1, bytelist(value, encodingName), cr)); + } + + public static CallSite emptyString(MethodHandles.Lookup lookup, String name, MethodType type, String encodingName) { + RubyString.EmptyByteListHolder holder = RubyString.getEmptyByteList(encodingFromName(encodingName)); + return new ConstantCallSite(insertArguments(STRING_HANDLE, 1, holder.bytes, holder.cr)); + } + + public static CallSite bufferString(MethodHandles.Lookup lookup, String name, MethodType type, String encodingName, int size) { + return new ConstantCallSite(insertArguments(BUFFERSTRING_HANDLE, 1, encodingFromName(encodingName), size, StringSupport.CR_7BIT)); + } + + public static CallSite fstring(MethodHandles.Lookup lookup, String name, MethodType type, String value, String encodingName, int cr, String file, int line) { + MutableCallSite site = new MutableCallSite(type); + + site.setTarget(insertArguments(FSTRING_HANDLE, 1, site, bytelist(value, encodingName), cr, file, line)); + + return site; + } + + public static RubyString string(ThreadContext context, ByteList value, int cr) { + return RubyString.newStringShared(context.runtime, value, cr); + } + + public static RubyString bufferString(ThreadContext context, Encoding encoding, int size, int cr) { + return RubyString.newString(context.runtime, new ByteList(size, encoding), cr); + } + + public static RubyString frozenString(ThreadContext context, MutableCallSite site, ByteList value, int cr, String file, int line) { + RubyString frozen = IRRuntimeHelpers.newFrozenString(context, value, cr, file, line); + + // Permanently bind to the new frozen string + site.setTarget(dropArguments(constant(RubyString.class, frozen), 0, ThreadContext.class)); + + return frozen; + } + + public static ByteList bytelist(String value, String encodingName) { + Encoding encoding = encodingFromName(encodingName); + + if (value.length() == 0) { + // special case empty string and don't create a new BL + return RubyString.getEmptyByteList(encoding).bytes; + } + + return new ByteList(RubyEncoding.encodeISO(value), encoding, false); + } + + public static ByteList bytelist(int size, String encodingName) { + Encoding encoding = encodingFromName(encodingName); + + return new ByteList(size, encoding); + } + + private static Encoding encodingFromName(String encodingName) { + Encoding encoding; + EncodingDB.Entry entry = EncodingDB.getEncodings().get(encodingName.getBytes()); + if (entry == null) entry = EncodingDB.getAliases().get(encodingName.getBytes()); + if (entry == null) throw new RuntimeException("could not find encoding: " + encodingName); + encoding = entry.getEncoding(); + return encoding; + } +} From aed7a408edeb6f80026f46f7df5db388006f1026 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 19 Oct 2023 17:22:55 -0500 Subject: [PATCH 03/56] Rehome indy accessors of runtime literals --- .../org/jruby/ir/targets/indy/Bootstrap.java | 164 +--------------- .../ir/targets/indy/IndyValueCompiler.java | 10 +- .../targets/indy/LiteralValueBootstrap.java | 175 ++++++++++++++++++ 3 files changed, 181 insertions(+), 168 deletions(-) create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/LiteralValueBootstrap.java diff --git a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java index 937a9d57416..dc463c4330e 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java @@ -30,14 +30,12 @@ import com.headius.invokebinder.Signature; import com.headius.invokebinder.SmartBinder; import com.headius.invokebinder.SmartHandle; -import org.jcodings.Encoding; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyBasicObject; import org.jruby.RubyBignum; import org.jruby.RubyBoolean; import org.jruby.RubyClass; -import org.jruby.RubyEncoding; import org.jruby.RubyFixnum; import org.jruby.RubyFloat; import org.jruby.RubyGlobal; @@ -80,7 +78,6 @@ import org.jruby.runtime.ivars.FieldVariableAccessor; import org.jruby.runtime.ivars.VariableAccessor; import org.jruby.runtime.opto.Invalidator; -import org.jruby.runtime.opto.OptoFactory; import org.jruby.runtime.scope.DynamicScopeGenerator; import org.jruby.specialized.RubyArraySpecialized; import org.jruby.util.ByteList; @@ -423,171 +420,12 @@ public static RubyArray array(ThreadContext context, IRubyObject[] ary) { return RubyArray.newArrayNoCopy(context.runtime, ary); } - public static Handle contextValue() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "contextValue", - sig(CallSite.class, Lookup.class, String.class, MethodType.class), - false); - } - - public static Handle contextValueString() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "contextValueString", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class), - false); - } - - public static CallSite contextValue(Lookup lookup, String name, MethodType type) { - MutableCallSite site = new MutableCallSite(type); - - MethodHandle dmh; - switch (name) { - case "runtime": - dmh = RUNTIME_HANDLE; - break; - case "nil": - dmh = NIL_HANDLE; - break; - case "True": - dmh = TRUE_HANDLE; - break; - case "False": - dmh = FALSE_HANDLE; - break; - case "rubyEncoding": - dmh = RUBY_ENCODING_HANDLE; - break; - case "encoding": - dmh = ENCODING_HANDLE; - break; - default: - throw new RuntimeException("BUG: invalid context value " + name); - } - - site.setTarget(Binder.from(type).append(site).invoke(dmh)); - - return site; - } - - public static CallSite contextValueString(Lookup lookup, String name, MethodType type, String str) { - MutableCallSite site = new MutableCallSite(type); - - MethodHandle dmh; - switch (name) { - case "rubyEncoding": - dmh = RUBY_ENCODING_HANDLE; - break; - case "encoding": - dmh = ENCODING_HANDLE; - break; - default: - throw new RuntimeException("BUG: invalid context value " + name); - } - - site.setTarget(Binder.from(type).append(site, str).invoke(dmh)); - return site; - } - - private static final MethodHandle RUNTIME_HANDLE = - Binder - .from(Ruby.class, ThreadContext.class, MutableCallSite.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "runtime"); - // We use LOOKUP here to have a full-featured MethodHandles.Lookup, avoiding jruby/jruby#7911 private static final MethodHandle RUNTIME_FROM_CONTEXT_HANDLE = Binder .from(LOOKUP, Ruby.class, ThreadContext.class) .getFieldQuiet("runtime"); - private static final MethodHandle NIL_HANDLE = - Binder - .from(IRubyObject.class, ThreadContext.class, MutableCallSite.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "nil"); - - private static final MethodHandle TRUE_HANDLE = - Binder - .from(IRubyObject.class, ThreadContext.class, MutableCallSite.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "True"); - - private static final MethodHandle FALSE_HANDLE = - Binder - .from(IRubyObject.class, ThreadContext.class, MutableCallSite.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "False"); - - private static final MethodHandle RUBY_ENCODING_HANDLE = - Binder - .from(RubyEncoding.class, ThreadContext.class, MutableCallSite.class, String.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "rubyEncoding"); - - private static final MethodHandle ENCODING_HANDLE = - Binder - .from(Encoding.class, ThreadContext.class, MutableCallSite.class, String.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "encoding"); - - public static IRubyObject nil(ThreadContext context, MutableCallSite site) { - RubyNil nil = (RubyNil) context.nil; - - MethodHandle constant = (MethodHandle) nil.constant(); - if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(IRubyObject.class, context.nil); - - site.setTarget(constant); - - return nil; - } - - public static IRubyObject True(ThreadContext context, MutableCallSite site) { - MethodHandle constant = (MethodHandle)context.tru.constant(); - if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(IRubyObject.class, context.tru); - - site.setTarget(constant); - - return context.tru; - } - - public static IRubyObject False(ThreadContext context, MutableCallSite site) { - MethodHandle constant = (MethodHandle)context.fals.constant(); - if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(IRubyObject.class, context.fals); - - site.setTarget(constant); - - return context.fals; - } - - public static Ruby runtime(ThreadContext context, MutableCallSite site) { - MethodHandle constant = (MethodHandle)context.runtime.constant(); - if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(Ruby.class, context.runtime); - - site.setTarget(constant); - - return context.runtime; - } - - public static RubyEncoding rubyEncoding(ThreadContext context, MutableCallSite site, String name) { - RubyEncoding rubyEncoding = IRRuntimeHelpers.retrieveEncoding(context, name); - - MethodHandle constant = (MethodHandle)rubyEncoding.constant(); - if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(RubyEncoding.class, rubyEncoding); - - site.setTarget(constant); - - return rubyEncoding; - } - - public static Encoding encoding(ThreadContext context, MutableCallSite site, String name) { - Encoding encoding = IRRuntimeHelpers.retrieveJCodingsEncoding(context, name); - - MethodHandle constant = MethodHandles.constant(Encoding.class, encoding); - if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(Encoding.class, encoding); - - site.setTarget(constant); - - return encoding; - } - public static RubyHash hash(ThreadContext context, IRubyObject[] pairs) { Ruby runtime = context.runtime; RubyHash hash = new RubyHash(runtime, pairs.length / 2 + 1); @@ -1455,7 +1293,7 @@ public static CallSite getHeapLocalOrNilBootstrap(Lookup lookup, String name, Me MethodHandle getter; Binder binder = Binder .from(type) - .filter(1, contextValue(lookup, "nil", methodType(IRubyObject.class, ThreadContext.class)).dynamicInvoker()); + .filter(1, LiteralValueBootstrap.contextValue(lookup, "nil", methodType(IRubyObject.class, ThreadContext.class)).dynamicInvoker()); if (depth == 0) { if (location < DynamicScopeGenerator.SPECIALIZED_GETS_OR_NIL.size()) { diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java index 75f86caa99f..014076e9641 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java @@ -40,7 +40,7 @@ public IndyValueCompiler(IRBytecodeAdapter compiler) { public void pushRuntime() { compiler.loadContext(); - compiler.adapter.invokedynamic("runtime", sig(Ruby.class, ThreadContext.class), Bootstrap.contextValue()); + compiler.adapter.invokedynamic("runtime", sig(Ruby.class, ThreadContext.class), LiteralValueBootstrap.CONTEXT_VALUE_HANDLE); } public void pushArrayClass() { compiler.loadContext(); @@ -119,22 +119,22 @@ public void pushSymbolProc(final ByteList bytes) { public void pushRubyEncoding(Encoding encoding) { compiler.loadContext(); - compiler.adapter.invokedynamic("rubyEncoding", sig(RubyEncoding.class, ThreadContext.class), Bootstrap.contextValueString(), new String(encoding.getName())); + compiler.adapter.invokedynamic("rubyEncoding", sig(RubyEncoding.class, ThreadContext.class), LiteralValueBootstrap.CONTEXT_VALUE_STRING_HANDLE, new String(encoding.getName())); } public void pushEncoding(Encoding encoding) { compiler.loadContext(); - compiler.adapter.invokedynamic("encoding", sig(RubyEncoding.class, ThreadContext.class), Bootstrap.contextValueString(), new String(encoding.getName())); + compiler.adapter.invokedynamic("encoding", sig(RubyEncoding.class, ThreadContext.class), LiteralValueBootstrap.CONTEXT_VALUE_STRING_HANDLE, new String(encoding.getName())); } public void pushNil() { compiler.loadContext(); - compiler.adapter.invokedynamic("nil", sig(IRubyObject.class, ThreadContext.class), Bootstrap.contextValue()); + compiler.adapter.invokedynamic("nil", sig(IRubyObject.class, ThreadContext.class), LiteralValueBootstrap.CONTEXT_VALUE_HANDLE); } public void pushBoolean(boolean b) { compiler.loadContext(); - compiler.adapter.invokedynamic(b ? "True" : "False", sig(IRubyObject.class, ThreadContext.class), Bootstrap.contextValue()); + compiler.adapter.invokedynamic(b ? "True" : "False", sig(IRubyObject.class, ThreadContext.class), LiteralValueBootstrap.CONTEXT_VALUE_HANDLE); } public void pushBignum(BigInteger bigint) { diff --git a/core/src/main/java/org/jruby/ir/targets/indy/LiteralValueBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/LiteralValueBootstrap.java new file mode 100644 index 00000000000..781fc15611c --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/LiteralValueBootstrap.java @@ -0,0 +1,175 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jcodings.Encoding; +import org.jruby.Ruby; +import org.jruby.RubyEncoding; +import org.jruby.RubyNil; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.opto.OptoFactory; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; + +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class LiteralValueBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + public static final Handle CONTEXT_VALUE_HANDLE = new Handle( + Opcodes.H_INVOKESTATIC, + p(LiteralValueBootstrap.class), + "contextValue", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class), + false); + public static final Handle CONTEXT_VALUE_STRING_HANDLE = new Handle( + Opcodes.H_INVOKESTATIC, + p(LiteralValueBootstrap.class), + "contextValueString", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class), + false); + + private static final MethodHandle RUNTIME_HANDLE = + Binder + .from(Ruby.class, ThreadContext.class, MutableCallSite.class) + .invokeStaticQuiet(LOOKUP, LiteralValueBootstrap.class, "runtime"); + private static final MethodHandle NIL_HANDLE = + Binder + .from(IRubyObject.class, ThreadContext.class, MutableCallSite.class) + .invokeStaticQuiet(LOOKUP, LiteralValueBootstrap.class, "nil"); + private static final MethodHandle TRUE_HANDLE = + Binder + .from(IRubyObject.class, ThreadContext.class, MutableCallSite.class) + .invokeStaticQuiet(LOOKUP, LiteralValueBootstrap.class, "True"); + private static final MethodHandle FALSE_HANDLE = + Binder + .from(IRubyObject.class, ThreadContext.class, MutableCallSite.class) + .invokeStaticQuiet(LOOKUP, LiteralValueBootstrap.class, "False"); + private static final MethodHandle RUBY_ENCODING_HANDLE = + Binder + .from(RubyEncoding.class, ThreadContext.class, MutableCallSite.class, String.class) + .invokeStaticQuiet(LOOKUP, LiteralValueBootstrap.class, "rubyEncoding"); + private static final MethodHandle ENCODING_HANDLE = + Binder + .from(Encoding.class, ThreadContext.class, MutableCallSite.class, String.class) + .invokeStaticQuiet(LOOKUP, LiteralValueBootstrap.class, "encoding"); + + public static CallSite contextValue(MethodHandles.Lookup lookup, String name, MethodType type) { + MutableCallSite site = new MutableCallSite(type); + + MethodHandle dmh; + switch (name) { + case "runtime": + dmh = RUNTIME_HANDLE; + break; + case "nil": + dmh = NIL_HANDLE; + break; + case "True": + dmh = TRUE_HANDLE; + break; + case "False": + dmh = FALSE_HANDLE; + break; + case "rubyEncoding": + dmh = RUBY_ENCODING_HANDLE; + break; + case "encoding": + dmh = ENCODING_HANDLE; + break; + default: + throw new RuntimeException("BUG: invalid context value " + name); + } + + site.setTarget(Binder.from(type).append(site).invoke(dmh)); + + return site; + } + + public static CallSite contextValueString(MethodHandles.Lookup lookup, String name, MethodType type, String str) { + MutableCallSite site = new MutableCallSite(type); + + MethodHandle dmh; + switch (name) { + case "rubyEncoding": + dmh = RUBY_ENCODING_HANDLE; + break; + case "encoding": + dmh = ENCODING_HANDLE; + break; + default: + throw new RuntimeException("BUG: invalid context value " + name); + } + + site.setTarget(Binder.from(type).append(site, str).invoke(dmh)); + return site; + } + + public static IRubyObject nil(ThreadContext context, MutableCallSite site) { + RubyNil nil = (RubyNil) context.nil; + + MethodHandle constant = (MethodHandle) nil.constant(); + if (constant == null) constant = (MethodHandle) OptoFactory.newConstantWrapper(IRubyObject.class, context.nil); + + site.setTarget(constant); + + return nil; + } + + public static IRubyObject True(ThreadContext context, MutableCallSite site) { + MethodHandle constant = (MethodHandle)context.tru.constant(); + if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(IRubyObject.class, context.tru); + + site.setTarget(constant); + + return context.tru; + } + + public static IRubyObject False(ThreadContext context, MutableCallSite site) { + MethodHandle constant = (MethodHandle)context.fals.constant(); + if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(IRubyObject.class, context.fals); + + site.setTarget(constant); + + return context.fals; + } + + public static Ruby runtime(ThreadContext context, MutableCallSite site) { + MethodHandle constant = (MethodHandle)context.runtime.constant(); + if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(Ruby.class, context.runtime); + + site.setTarget(constant); + + return context.runtime; + } + + public static RubyEncoding rubyEncoding(ThreadContext context, MutableCallSite site, String name) { + RubyEncoding rubyEncoding = IRRuntimeHelpers.retrieveEncoding(context, name); + + MethodHandle constant = (MethodHandle)rubyEncoding.constant(); + if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(RubyEncoding.class, rubyEncoding); + + site.setTarget(constant); + + return rubyEncoding; + } + + public static Encoding encoding(ThreadContext context, MutableCallSite site, String name) { + Encoding encoding = IRRuntimeHelpers.retrieveJCodingsEncoding(context, name); + + MethodHandle constant = MethodHandles.constant(Encoding.class, encoding); + if (constant == null) constant = (MethodHandle)OptoFactory.newConstantWrapper(Encoding.class, encoding); + + site.setTarget(constant); + + return encoding; + } +} From ad125eab91084624dec4252e97df880c869d3354 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 19 Oct 2023 17:32:26 -0500 Subject: [PATCH 04/56] Move Java indy dispatch logic to JavaBootstrap --- .../org/jruby/ir/targets/indy/Bootstrap.java | 268 +--------------- .../jruby/ir/targets/indy/JavaBootstrap.java | 292 ++++++++++++++++++ 2 files changed, 293 insertions(+), 267 deletions(-) create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/JavaBootstrap.java diff --git a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java index dc463c4330e..901bc02fbce 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java @@ -33,16 +33,12 @@ import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyBasicObject; -import org.jruby.RubyBignum; import org.jruby.RubyBoolean; import org.jruby.RubyClass; -import org.jruby.RubyFixnum; -import org.jruby.RubyFloat; import org.jruby.RubyGlobal; import org.jruby.RubyHash; import org.jruby.RubyModule; import org.jruby.RubyNil; -import org.jruby.RubyString; import org.jruby.anno.JRubyMethod; import org.jruby.internal.runtime.GlobalVariable; import org.jruby.internal.runtime.methods.AttrReaderMethod; @@ -54,8 +50,6 @@ import org.jruby.internal.runtime.methods.NativeCallMethod; import org.jruby.ir.JIT; import org.jruby.ir.runtime.IRRuntimeHelpers; -import org.jruby.java.invokers.SingletonMethodInvoker; -import org.jruby.javasupport.JavaUtil; import org.jruby.javasupport.proxy.ReifiedJavaProxy; import org.jruby.parser.StaticScope; import org.jruby.runtime.Binding; @@ -80,8 +74,6 @@ import org.jruby.runtime.opto.Invalidator; import org.jruby.runtime.scope.DynamicScopeGenerator; import org.jruby.specialized.RubyArraySpecialized; -import org.jruby.util.ByteList; -import org.jruby.util.CodegenUtils; import org.jruby.util.JavaNameMangler; import org.jruby.util.cli.Options; import org.jruby.util.log.Logger; @@ -96,20 +88,16 @@ import java.lang.invoke.MethodType; import java.lang.invoke.MutableCallSite; import java.lang.invoke.SwitchPoint; -import java.math.BigInteger; import static java.lang.invoke.MethodHandles.Lookup; import static java.lang.invoke.MethodHandles.constant; import static java.lang.invoke.MethodHandles.dropArguments; -import static java.lang.invoke.MethodHandles.explicitCastArguments; -import static java.lang.invoke.MethodHandles.filterArguments; import static java.lang.invoke.MethodHandles.insertArguments; import static java.lang.invoke.MethodHandles.lookup; import static java.lang.invoke.MethodType.methodType; import static org.jruby.runtime.Helpers.arrayOf; import static org.jruby.runtime.Helpers.constructObjectArrayHandle; import static org.jruby.runtime.invokedynamic.InvokeDynamicSupport.findStatic; -import static org.jruby.runtime.invokedynamic.InvokeDynamicSupport.findVirtual; import static org.jruby.util.CodegenUtils.p; import static org.jruby.util.CodegenUtils.sig; @@ -775,7 +763,7 @@ static MethodHandle buildNativeHandle(InvokeSite site, CacheEntry entry, boolean DynamicMethod.NativeCall nc = nativeCall; if (nc.isJava()) { - return createJavaHandle(site, method); + return JavaBootstrap.createJavaHandle(site, method); } else { int nativeArgCount = getNativeArgCount(method, nativeCall); @@ -905,260 +893,6 @@ public static boolean testType(RubyClass original, IRubyObject self) { return original == RubyBasicObject.getMetaClass(self); } - - //////////////////////////////////////////////////////////////////////////// - // Dispatch from Ruby to Java via Java integration - //////////////////////////////////////////////////////////////////////////// - - private static MethodHandle createJavaHandle(InvokeSite site, DynamicMethod method) { - if (!(method instanceof NativeCallMethod)) return null; - - DynamicMethod.NativeCall nativeCall = ((NativeCallMethod) method).getNativeCall(); - - if (nativeCall == null) return null; - - boolean isStatic = nativeCall.isStatic(); - - // This logic does not handle closure conversion yet - if (site.signature.lastArgType() == Block.class) { - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tpassed a closure to Java method " + nativeCall + ": " + Bootstrap.logMethod(method)); - } - return null; - } - - // mismatched arity not supported - if (site.arity != nativeCall.getNativeSignature().length) { - return null; - } - - // varargs broken, so ignore methods that take a trailing array - Class[] signature = nativeCall.getNativeSignature(); - if (signature.length > 0 && signature[signature.length - 1].isArray()) { - return null; - } - - // Scala singletons have slightly different JI logic, so skip for now - if (method instanceof SingletonMethodInvoker) return null; - - // get prepared handle if previously cached - MethodHandle nativeTarget = (MethodHandle)method.getHandle(); - - if (nativeTarget == null) { - // no cache, proceed to construct the adapted handle - - // the "apparent" type from the NativeCall, excluding receiver - MethodType apparentType = methodType(nativeCall.getNativeReturn(), nativeCall.getNativeSignature()); - - if (isStatic) { - nativeTarget = findStatic(nativeCall.getNativeTarget(), nativeCall.getNativeName(), apparentType); - } else { - nativeTarget = findVirtual(nativeCall.getNativeTarget(), nativeCall.getNativeName(), apparentType); - } - - // the actual native type with receiver - MethodType nativeType = nativeTarget.type(); - Class[] nativeParams = nativeType.parameterArray(); - Class nativeReturn = nativeType.returnType(); - - // convert arguments - MethodHandle[] argConverters = new MethodHandle[nativeType.parameterCount()]; - for (int i = 0; i < argConverters.length; i++) { - MethodHandle converter; - if (!isStatic && i == 0) { - // handle non-static receiver specially - converter = Binder - .from(nativeParams[0], IRubyObject.class) - .cast(Object.class, IRubyObject.class) - .invokeStaticQuiet(lookup(), JavaUtil.class, "objectFromJavaProxy"); - } else { - // all other arguments use toJava - converter = Binder - .from(nativeParams[i], IRubyObject.class) - .insert(1, nativeParams[i]) - .cast(Object.class, IRubyObject.class, Class.class) - .invokeVirtualQuiet(lookup(), "toJava"); - } - argConverters[i] = converter; - } - nativeTarget = filterArguments(nativeTarget, 0, argConverters); - - Ruby runtime = method.getImplementationClass().getRuntime(); - Class[] convertedParams = CodegenUtils.params(IRubyObject.class, nativeTarget.type().parameterCount()); - - // handle return value - MethodHandle returnFilter; - if (nativeReturn == byte.class - || nativeReturn == short.class - || nativeReturn == char.class - || nativeReturn == int.class - || nativeReturn == long.class) { - // native integral type, produce a Fixnum - nativeTarget = explicitCastArguments(nativeTarget, methodType(long.class, convertedParams)); - returnFilter = insertArguments( - findStatic(RubyFixnum.class, "newFixnum", methodType(RubyFixnum.class, Ruby.class, long.class)), - 0, - runtime); - } else if (nativeReturn == Byte.class - || nativeReturn == Short.class - || nativeReturn == Character.class - || nativeReturn == Integer.class - || nativeReturn == Long.class) { - // boxed integral type, produce a Fixnum or nil - returnFilter = insertArguments( - findStatic(Bootstrap.class, "fixnumOrNil", methodType(IRubyObject.class, Ruby.class, nativeReturn)), - 0, - runtime); - } else if (nativeReturn == float.class - || nativeReturn == double.class) { - // native decimal type, produce a Float - nativeTarget = explicitCastArguments(nativeTarget, methodType(double.class, convertedParams)); - returnFilter = insertArguments( - findStatic(RubyFloat.class, "newFloat", methodType(RubyFloat.class, Ruby.class, double.class)), - 0, - runtime); - } else if (nativeReturn == Float.class - || nativeReturn == Double.class) { - // boxed decimal type, produce a Float or nil - returnFilter = insertArguments( - findStatic(Bootstrap.class, "floatOrNil", methodType(IRubyObject.class, Ruby.class, nativeReturn)), - 0, - runtime); - } else if (nativeReturn == boolean.class) { - // native boolean type, produce a Boolean - nativeTarget = explicitCastArguments(nativeTarget, methodType(boolean.class, convertedParams)); - returnFilter = insertArguments( - findStatic(RubyBoolean.class, "newBoolean", methodType(RubyBoolean.class, Ruby.class, boolean.class)), - 0, - runtime); - } else if (nativeReturn == Boolean.class) { - // boxed boolean type, produce a Boolean or nil - returnFilter = insertArguments( - findStatic(Bootstrap.class, "booleanOrNil", methodType(IRubyObject.class, Ruby.class, Boolean.class)), - 0, - runtime); - } else if (CharSequence.class.isAssignableFrom(nativeReturn)) { - // character sequence, produce a String or nil - nativeTarget = explicitCastArguments(nativeTarget, methodType(CharSequence.class, convertedParams)); - returnFilter = insertArguments( - findStatic(Bootstrap.class, "stringOrNil", methodType(IRubyObject.class, Ruby.class, CharSequence.class)), - 0, - runtime); - } else if (nativeReturn == void.class) { - // void return, produce nil - returnFilter = constant(IRubyObject.class, runtime.getNil()); - } else if (nativeReturn == ByteList.class) { - // bytelist, produce a String or nil - nativeTarget = explicitCastArguments(nativeTarget, methodType(ByteList.class, convertedParams)); - returnFilter = insertArguments( - findStatic(Bootstrap.class, "stringOrNil", methodType(IRubyObject.class, Ruby.class, ByteList.class)), - 0, - runtime); - } else if (nativeReturn == BigInteger.class) { - // bytelist, produce a String or nil - nativeTarget = explicitCastArguments(nativeTarget, methodType(BigInteger.class, convertedParams)); - returnFilter = insertArguments( - findStatic(Bootstrap.class, "bignumOrNil", methodType(IRubyObject.class, Ruby.class, BigInteger.class)), - 0, - runtime); - } else { - // all other object types - nativeTarget = explicitCastArguments(nativeTarget, methodType(Object.class, convertedParams)); - returnFilter = insertArguments( - findStatic(JavaUtil.class, "convertJavaToUsableRubyObject", methodType(IRubyObject.class, Ruby.class, Object.class)), - 0, - runtime); - } - - nativeTarget = MethodHandles.filterReturnValue(nativeTarget, returnFilter); - - // cache adapted handle - method.setHandle(nativeTarget); - } - - // perform final adaptation by dropping JRuby-specific arguments - int dropCount; - if (isStatic) { - if (site.functional) { - dropCount = 2; // context, self - } else { - dropCount = 3; // context, caller, self - } - } else { - if (site.functional) { - dropCount = 1; // context - } else { - dropCount = 2; // context, caller - } - } - - return Binder - .from(site.type()) - .drop(0, dropCount) - .invoke(nativeTarget); - } - - public static boolean subclassProxyTest(Object target) { - return target instanceof ReifiedJavaProxy; - } - - private static final MethodHandle IS_JAVA_SUBCLASS = findStatic(Bootstrap.class, "subclassProxyTest", methodType(boolean.class, Object.class)); - - private static Object nullValue(Class type) { - if (type == boolean.class || type == Boolean.class) return false; - if (type == byte.class || type == Byte.class) return (byte)0; - if (type == short.class || type == Short.class) return (short)0; - if (type == int.class || type == Integer.class) return 0; - if (type == long.class || type == Long.class) return 0L; - if (type == float.class || type == Float.class) return 0.0F; - if (type == double.class || type == Double.class)return 0.0; - return null; - } - - public static IRubyObject fixnumOrNil(Ruby runtime, Byte b) { - return b == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, b); - } - - public static IRubyObject fixnumOrNil(Ruby runtime, Short s) { - return s == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, s); - } - - public static IRubyObject fixnumOrNil(Ruby runtime, Character c) { - return c == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, c); - } - - public static IRubyObject fixnumOrNil(Ruby runtime, Integer i) { - return i == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, i); - } - - public static IRubyObject fixnumOrNil(Ruby runtime, Long l) { - return l == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, l); - } - - public static IRubyObject floatOrNil(Ruby runtime, Float f) { - return f == null ? runtime.getNil() : RubyFloat.newFloat(runtime, f); - } - - public static IRubyObject floatOrNil(Ruby runtime, Double d) { - return d == null ? runtime.getNil() : RubyFloat.newFloat(runtime, d); - } - - public static IRubyObject booleanOrNil(Ruby runtime, Boolean b) { - return b == null ? runtime.getNil() : RubyBoolean.newBoolean(runtime, b); - } - - public static IRubyObject stringOrNil(Ruby runtime, CharSequence cs) { - return cs == null ? runtime.getNil() : RubyString.newUnicodeString(runtime, cs); - } - - public static IRubyObject stringOrNil(Ruby runtime, ByteList bl) { - return bl == null ? runtime.getNil() : RubyString.newString(runtime, bl); - } - - public static IRubyObject bignumOrNil(Ruby runtime, BigInteger bi) { - return bi == null ? runtime.getNil() : RubyBignum.newBignum(runtime, bi); - } - /////////////////////////////////////////////////////////////////////////// // Fixnum binding diff --git a/core/src/main/java/org/jruby/ir/targets/indy/JavaBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/JavaBootstrap.java new file mode 100644 index 00000000000..0b4c937ae51 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/JavaBootstrap.java @@ -0,0 +1,292 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.Ruby; +import org.jruby.RubyBignum; +import org.jruby.RubyBoolean; +import org.jruby.RubyFixnum; +import org.jruby.RubyFloat; +import org.jruby.RubyString; +import org.jruby.internal.runtime.methods.DynamicMethod; +import org.jruby.internal.runtime.methods.NativeCallMethod; +import org.jruby.java.invokers.SingletonMethodInvoker; +import org.jruby.javasupport.JavaUtil; +import org.jruby.javasupport.proxy.ReifiedJavaProxy; +import org.jruby.runtime.Block; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.ByteList; +import org.jruby.util.CodegenUtils; +import org.jruby.util.cli.Options; +import org.jruby.util.log.Logger; +import org.jruby.util.log.LoggerFactory; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.math.BigInteger; + +import static java.lang.invoke.MethodHandles.constant; +import static java.lang.invoke.MethodHandles.explicitCastArguments; +import static java.lang.invoke.MethodHandles.filterArguments; +import static java.lang.invoke.MethodHandles.insertArguments; +import static java.lang.invoke.MethodHandles.lookup; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.runtime.invokedynamic.InvokeDynamicSupport.findStatic; +import static org.jruby.runtime.invokedynamic.InvokeDynamicSupport.findVirtual; + +public class JavaBootstrap { + //////////////////////////////////////////////////////////////////////////// + // Dispatch from Ruby to Java via Java integration + //////////////////////////////////////////////////////////////////////////// + + private static final Logger LOG = LoggerFactory.getLogger(JavaBootstrap.class); + + public static boolean subclassProxyTest(Object target) { + return target instanceof ReifiedJavaProxy; + } + + private static final MethodHandle IS_JAVA_SUBCLASS = findStatic(JavaBootstrap.class, "subclassProxyTest", methodType(boolean.class, Object.class)); + + private static Object nullValue(Class type) { + if (type == boolean.class || type == Boolean.class) return false; + if (type == byte.class || type == Byte.class) return (byte)0; + if (type == short.class || type == Short.class) return (short)0; + if (type == int.class || type == Integer.class) return 0; + if (type == long.class || type == Long.class) return 0L; + if (type == float.class || type == Float.class) return 0.0F; + if (type == double.class || type == Double.class)return 0.0; + return null; + } + + public static MethodHandle createJavaHandle(InvokeSite site, DynamicMethod method) { + if (!(method instanceof NativeCallMethod)) return null; + + DynamicMethod.NativeCall nativeCall = ((NativeCallMethod) method).getNativeCall(); + + if (nativeCall == null) return null; + + boolean isStatic = nativeCall.isStatic(); + + // This logic does not handle closure conversion yet + if (site.signature.lastArgType() == Block.class) { + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(site.name() + "\tpassed a closure to Java method " + nativeCall + ": " + Bootstrap.logMethod(method)); + } + return null; + } + + // mismatched arity not supported + if (site.arity != nativeCall.getNativeSignature().length) { + return null; + } + + // varargs broken, so ignore methods that take a trailing array + Class[] signature = nativeCall.getNativeSignature(); + if (signature.length > 0 && signature[signature.length - 1].isArray()) { + return null; + } + + // Scala singletons have slightly different JI logic, so skip for now + if (method instanceof SingletonMethodInvoker) return null; + + // get prepared handle if previously cached + MethodHandle nativeTarget = (MethodHandle)method.getHandle(); + + if (nativeTarget == null) { + // no cache, proceed to construct the adapted handle + + // the "apparent" type from the NativeCall, excluding receiver + MethodType apparentType = methodType(nativeCall.getNativeReturn(), nativeCall.getNativeSignature()); + + if (isStatic) { + nativeTarget = findStatic(nativeCall.getNativeTarget(), nativeCall.getNativeName(), apparentType); + } else { + nativeTarget = findVirtual(nativeCall.getNativeTarget(), nativeCall.getNativeName(), apparentType); + } + + // the actual native type with receiver + MethodType nativeType = nativeTarget.type(); + Class[] nativeParams = nativeType.parameterArray(); + Class nativeReturn = nativeType.returnType(); + + // convert arguments + MethodHandle[] argConverters = new MethodHandle[nativeType.parameterCount()]; + for (int i = 0; i < argConverters.length; i++) { + MethodHandle converter; + if (!isStatic && i == 0) { + // handle non-static receiver specially + converter = Binder + .from(nativeParams[0], IRubyObject.class) + .cast(Object.class, IRubyObject.class) + .invokeStaticQuiet(lookup(), JavaUtil.class, "objectFromJavaProxy"); + } else { + // all other arguments use toJava + converter = Binder + .from(nativeParams[i], IRubyObject.class) + .insert(1, nativeParams[i]) + .cast(Object.class, IRubyObject.class, Class.class) + .invokeVirtualQuiet(lookup(), "toJava"); + } + argConverters[i] = converter; + } + nativeTarget = filterArguments(nativeTarget, 0, argConverters); + + Ruby runtime = method.getImplementationClass().getRuntime(); + Class[] convertedParams = CodegenUtils.params(IRubyObject.class, nativeTarget.type().parameterCount()); + + // handle return value + MethodHandle returnFilter; + if (nativeReturn == byte.class + || nativeReturn == short.class + || nativeReturn == char.class + || nativeReturn == int.class + || nativeReturn == long.class) { + // native integral type, produce a Fixnum + nativeTarget = explicitCastArguments(nativeTarget, methodType(long.class, convertedParams)); + returnFilter = insertArguments( + findStatic(RubyFixnum.class, "newFixnum", methodType(RubyFixnum.class, Ruby.class, long.class)), + 0, + runtime); + } else if (nativeReturn == Byte.class + || nativeReturn == Short.class + || nativeReturn == Character.class + || nativeReturn == Integer.class + || nativeReturn == Long.class) { + // boxed integral type, produce a Fixnum or nil + returnFilter = insertArguments( + findStatic(JavaBootstrap.class, "fixnumOrNil", methodType(IRubyObject.class, Ruby.class, nativeReturn)), + 0, + runtime); + } else if (nativeReturn == float.class + || nativeReturn == double.class) { + // native decimal type, produce a Float + nativeTarget = explicitCastArguments(nativeTarget, methodType(double.class, convertedParams)); + returnFilter = insertArguments( + findStatic(RubyFloat.class, "newFloat", methodType(RubyFloat.class, Ruby.class, double.class)), + 0, + runtime); + } else if (nativeReturn == Float.class + || nativeReturn == Double.class) { + // boxed decimal type, produce a Float or nil + returnFilter = insertArguments( + findStatic(JavaBootstrap.class, "floatOrNil", methodType(IRubyObject.class, Ruby.class, nativeReturn)), + 0, + runtime); + } else if (nativeReturn == boolean.class) { + // native boolean type, produce a Boolean + nativeTarget = explicitCastArguments(nativeTarget, methodType(boolean.class, convertedParams)); + returnFilter = insertArguments( + findStatic(RubyBoolean.class, "newBoolean", methodType(RubyBoolean.class, Ruby.class, boolean.class)), + 0, + runtime); + } else if (nativeReturn == Boolean.class) { + // boxed boolean type, produce a Boolean or nil + returnFilter = insertArguments( + findStatic(JavaBootstrap.class, "booleanOrNil", methodType(IRubyObject.class, Ruby.class, Boolean.class)), + 0, + runtime); + } else if (CharSequence.class.isAssignableFrom(nativeReturn)) { + // character sequence, produce a String or nil + nativeTarget = explicitCastArguments(nativeTarget, methodType(CharSequence.class, convertedParams)); + returnFilter = insertArguments( + findStatic(JavaBootstrap.class, "stringOrNil", methodType(IRubyObject.class, Ruby.class, CharSequence.class)), + 0, + runtime); + } else if (nativeReturn == void.class) { + // void return, produce nil + returnFilter = constant(IRubyObject.class, runtime.getNil()); + } else if (nativeReturn == ByteList.class) { + // bytelist, produce a String or nil + nativeTarget = explicitCastArguments(nativeTarget, methodType(ByteList.class, convertedParams)); + returnFilter = insertArguments( + findStatic(JavaBootstrap.class, "stringOrNil", methodType(IRubyObject.class, Ruby.class, ByteList.class)), + 0, + runtime); + } else if (nativeReturn == BigInteger.class) { + // bytelist, produce a String or nil + nativeTarget = explicitCastArguments(nativeTarget, methodType(BigInteger.class, convertedParams)); + returnFilter = insertArguments( + findStatic(JavaBootstrap.class, "bignumOrNil", methodType(IRubyObject.class, Ruby.class, BigInteger.class)), + 0, + runtime); + } else { + // all other object types + nativeTarget = explicitCastArguments(nativeTarget, methodType(Object.class, convertedParams)); + returnFilter = insertArguments( + findStatic(JavaUtil.class, "convertJavaToUsableRubyObject", methodType(IRubyObject.class, Ruby.class, Object.class)), + 0, + runtime); + } + + nativeTarget = MethodHandles.filterReturnValue(nativeTarget, returnFilter); + + // cache adapted handle + method.setHandle(nativeTarget); + } + + // perform final adaptation by dropping JRuby-specific arguments + int dropCount; + if (isStatic) { + if (site.functional) { + dropCount = 2; // context, self + } else { + dropCount = 3; // context, caller, self + } + } else { + if (site.functional) { + dropCount = 1; // context + } else { + dropCount = 2; // context, caller + } + } + + return Binder + .from(site.type()) + .drop(0, dropCount) + .invoke(nativeTarget); + } + + public static IRubyObject fixnumOrNil(Ruby runtime, Byte b) { + return b == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, b); + } + + public static IRubyObject fixnumOrNil(Ruby runtime, Short s) { + return s == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, s); + } + + public static IRubyObject fixnumOrNil(Ruby runtime, Character c) { + return c == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, c); + } + + public static IRubyObject fixnumOrNil(Ruby runtime, Integer i) { + return i == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, i); + } + + public static IRubyObject fixnumOrNil(Ruby runtime, Long l) { + return l == null ? runtime.getNil() : RubyFixnum.newFixnum(runtime, l); + } + + public static IRubyObject floatOrNil(Ruby runtime, Float f) { + return f == null ? runtime.getNil() : RubyFloat.newFloat(runtime, f); + } + + public static IRubyObject floatOrNil(Ruby runtime, Double d) { + return d == null ? runtime.getNil() : RubyFloat.newFloat(runtime, d); + } + + public static IRubyObject booleanOrNil(Ruby runtime, Boolean b) { + return b == null ? runtime.getNil() : RubyBoolean.newBoolean(runtime, b); + } + + public static IRubyObject stringOrNil(Ruby runtime, CharSequence cs) { + return cs == null ? runtime.getNil() : RubyString.newUnicodeString(runtime, cs); + } + + public static IRubyObject stringOrNil(Ruby runtime, ByteList bl) { + return bl == null ? runtime.getNil() : RubyString.newString(runtime, bl); + } + + public static IRubyObject bignumOrNil(Ruby runtime, BigInteger bi) { + return bi == null ? runtime.getNil() : RubyBignum.newBignum(runtime, bi); + } +} From f21f7d5ec163ef504db8b138b778a8df8ba5b445 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 19 Oct 2023 18:17:39 -0500 Subject: [PATCH 05/56] Move more bootstrap to logical homes --- .../jruby/ir/targets/indy/ArrayBootstrap.java | 48 + .../org/jruby/ir/targets/indy/Bootstrap.java | 844 +----------------- .../ir/targets/indy/CheckArityBootstrap.java | 76 ++ .../ir/targets/indy/ConstantLookupSite.java | 14 +- .../jruby/ir/targets/indy/HashBootstrap.java | 99 ++ .../targets/indy/IndyArgumentsCompiler.java | 2 +- .../ir/targets/indy/IndyBranchCompiler.java | 10 +- .../indy/IndyDynamicValueCompiler.java | 4 +- .../indy/IndyGlobalVariableCompiler.java | 5 +- .../targets/indy/IndyInvocationCompiler.java | 16 +- .../org/jruby/ir/targets/indy/InvokeSite.java | 510 ++++++++++- .../org/jruby/ir/targets/indy/IsNilSite.java | 60 ++ .../org/jruby/ir/targets/indy/IsTrueSite.java | 64 ++ .../targets/simple/NormalBranchCompiler.java | 6 +- .../runtime/invokedynamic/GlobalSite.java | 109 ++- 15 files changed, 978 insertions(+), 889 deletions(-) create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/ArrayBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/CheckArityBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/HashBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/IsNilSite.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/IsTrueSite.java diff --git a/core/src/main/java/org/jruby/ir/targets/indy/ArrayBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/ArrayBootstrap.java new file mode 100644 index 00000000000..a3ffbe429e9 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/ArrayBootstrap.java @@ -0,0 +1,48 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.RubyArray; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.specialized.RubyArraySpecialized; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class ArrayBootstrap { + public static final Handle ARRAY_H = new Handle( + Opcodes.H_INVOKESTATIC, + p(ArrayBootstrap.class), + "array", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class), + false); + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + private static final MethodHandle ARRAY_HANDLE = + Binder + .from(RubyArray.class, ThreadContext.class, IRubyObject[].class) + .invokeStaticQuiet(LOOKUP, ArrayBootstrap.class, "array"); + + public static CallSite array(MethodHandles.Lookup lookup, String name, MethodType type) { + MethodHandle handle = Binder + .from(type) + .collect(1, IRubyObject[].class) + .invoke(ARRAY_HANDLE); + + return new ConstantCallSite(handle); + } + + public static RubyArray array(ThreadContext context, IRubyObject[] ary) { + assert ary.length > RubyArraySpecialized.MAX_PACKED_SIZE; + // array() only dispatches here if ^^ holds + return RubyArray.newArrayNoCopy(context.runtime, ary); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java index 901bc02fbce..1aac20c8cd8 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java @@ -27,30 +27,11 @@ package org.jruby.ir.targets.indy; import com.headius.invokebinder.Binder; -import com.headius.invokebinder.Signature; -import com.headius.invokebinder.SmartBinder; -import com.headius.invokebinder.SmartHandle; import org.jruby.Ruby; -import org.jruby.RubyArray; -import org.jruby.RubyBasicObject; -import org.jruby.RubyBoolean; -import org.jruby.RubyClass; -import org.jruby.RubyGlobal; -import org.jruby.RubyHash; import org.jruby.RubyModule; -import org.jruby.RubyNil; -import org.jruby.anno.JRubyMethod; -import org.jruby.internal.runtime.GlobalVariable; -import org.jruby.internal.runtime.methods.AttrReaderMethod; -import org.jruby.internal.runtime.methods.AttrWriterMethod; -import org.jruby.internal.runtime.methods.CompiledIRMethod; import org.jruby.internal.runtime.methods.DynamicMethod; -import org.jruby.internal.runtime.methods.HandleMethod; -import org.jruby.internal.runtime.methods.MixedModeIRMethod; -import org.jruby.internal.runtime.methods.NativeCallMethod; import org.jruby.ir.JIT; import org.jruby.ir.runtime.IRRuntimeHelpers; -import org.jruby.javasupport.proxy.ReifiedJavaProxy; import org.jruby.parser.StaticScope; import org.jruby.runtime.Binding; import org.jruby.runtime.Block; @@ -61,21 +42,13 @@ import org.jruby.runtime.Helpers; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; -import org.jruby.runtime.callsite.CacheEntry; import org.jruby.runtime.callsite.CachingCallSite; import org.jruby.runtime.callsite.FunctionalCachingCallSite; import org.jruby.runtime.callsite.MonomorphicCallSite; import org.jruby.runtime.callsite.VariableCachingCallSite; -import org.jruby.runtime.invokedynamic.GlobalSite; -import org.jruby.runtime.invokedynamic.InvocationLinker; import org.jruby.runtime.invokedynamic.MathLinker; -import org.jruby.runtime.ivars.FieldVariableAccessor; -import org.jruby.runtime.ivars.VariableAccessor; import org.jruby.runtime.opto.Invalidator; import org.jruby.runtime.scope.DynamicScopeGenerator; -import org.jruby.specialized.RubyArraySpecialized; -import org.jruby.util.JavaNameMangler; -import org.jruby.util.cli.Options; import org.jruby.util.log.Logger; import org.jruby.util.log.LoggerFactory; import org.objectweb.asm.Handle; @@ -96,8 +69,6 @@ import static java.lang.invoke.MethodHandles.lookup; import static java.lang.invoke.MethodType.methodType; import static org.jruby.runtime.Helpers.arrayOf; -import static org.jruby.runtime.Helpers.constructObjectArrayHandle; -import static org.jruby.runtime.invokedynamic.InvokeDynamicSupport.findStatic; import static org.jruby.util.CodegenUtils.p; import static org.jruby.util.CodegenUtils.sig; @@ -108,99 +79,7 @@ public class Bootstrap { public final static String BOOTSTRAP_INT_INT_SIG = sig(CallSite.class, Lookup.class, String.class, MethodType.class, int.class, int.class); public final static String BOOTSTRAP_INT_SIG = sig(CallSite.class, Lookup.class, String.class, MethodType.class, int.class); private static final Logger LOG = LoggerFactory.getLogger(Bootstrap.class); - static final Lookup LOOKUP = MethodHandles.lookup(); - private static final String[] GENERIC_CALL_PERMUTE = {"context", "self", "arg.*"}; - - - public static Handle isNilBoot() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "isNil", - sig(CallSite.class, Lookup.class, String.class, MethodType.class), - false); - } - - public static CallSite isNil(Lookup lookup, String name, MethodType type) { - return new IsNilSite(); - } - - public static class IsNilSite extends MutableCallSite { - - public static final MethodType TYPE = methodType(boolean.class, IRubyObject.class); - - private static final MethodHandle INIT_HANDLE = - Binder.from(TYPE.insertParameterTypes(0, IsNilSite.class)).invokeVirtualQuiet(LOOKUP, "init"); - - private static final MethodHandle IS_NIL_HANDLE = - Binder - .from(boolean.class, IRubyObject.class, RubyNil.class) - .invokeStaticQuiet(LOOKUP, IsNilSite.class, "isNil"); - - public IsNilSite() { - super(TYPE); - - setTarget(INIT_HANDLE.bindTo(this)); - } - - public boolean init(IRubyObject obj) { - IRubyObject nil = obj.getRuntime().getNil(); - - setTarget(insertArguments(IS_NIL_HANDLE, 1, nil)); - - return nil == obj; - } - - public static boolean isNil(IRubyObject obj, RubyNil nil) { - return nil == obj; - } - } - - public static Handle isTrueBoot() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "isTrue", - sig(CallSite.class, Lookup.class, String.class, MethodType.class), - false); - } - - public static CallSite isTrue(Lookup lookup, String name, MethodType type) { - return new IsTrueSite(); - } - - public static class IsTrueSite extends MutableCallSite { - - public static final MethodType TYPE = methodType(boolean.class, IRubyObject.class); - - public static final MethodHandle INIT_HANDLE = Binder.from(TYPE.insertParameterTypes(0, IsTrueSite.class)).invokeVirtualQuiet(LOOKUP, "init"); - - private static final MethodHandle IS_TRUE_HANDLE = - Binder - .from(boolean.class, IRubyObject.class, RubyNil.class, RubyBoolean.False.class) - .invokeStaticQuiet(LOOKUP, IsTrueSite.class, "isTruthy"); - - public IsTrueSite() { - super(TYPE); - - setTarget(INIT_HANDLE.bindTo(this)); - } - - public boolean init(IRubyObject obj) { - Ruby runtime = obj.getRuntime(); - - IRubyObject nil = runtime.getNil(); - IRubyObject fals = runtime.getFalse(); - - setTarget(insertArguments(IS_TRUE_HANDLE, 1, nil, fals)); - - return nil != obj && fals != obj; - } - - public static boolean isTruthy(IRubyObject obj, RubyNil nil, RubyBoolean.False fals) { - return nil != obj && fals != obj; - } - } + private static final Lookup LOOKUP = MethodHandles.lookup(); public static final Handle CALLSITE = new Handle( Opcodes.H_INVOKESTATIC, @@ -258,641 +137,6 @@ public static DynamicMethod openMetaClass(ThreadContext context, IRubyObject obj return IRRuntimeHelpers.newCompiledMetaClass(context, body, scope, object, line, dynscopeEliminated, refinements); } - public static final Handle CHECK_ARITY_SPECIFIC_ARGS = new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "checkAritySpecificArgs", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, int.class, int.class, int.class, int.class), - false); - - public static final Handle CHECK_ARITY = new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "checkArity", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, int.class, int.class, int.class, int.class), - false); - - @JIT - public static CallSite checkArity(Lookup lookup, String name, MethodType type, int req, int opt, int rest, int keyrest) { - return new ConstantCallSite(insertArguments(CHECK_ARITY_HANDLE, 5, req, opt, rest == 0 ? false : true, keyrest)); - } - - private static final MethodHandle CHECK_ARITY_HANDLE = - Binder - .from(void.class, ThreadContext.class, StaticScope.class, Object[].class, Object.class, Block.class, int.class, int.class, boolean.class, int.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "checkArity"); - - @JIT - public static CallSite checkAritySpecificArgs(Lookup lookup, String name, MethodType type, int req, int opt, int rest, int keyrest) { - return new ConstantCallSite(insertArguments(CHECK_ARITY_SPECIFIC_ARGS_HANDLE, 4, req, opt, rest == 0 ? false : true, keyrest)); - } - - private static final MethodHandle CHECK_ARITY_SPECIFIC_ARGS_HANDLE = - Binder - .from(void.class, ThreadContext.class, StaticScope.class, Object[].class, Block.class, int.class, int.class, boolean.class, int.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "checkAritySpecificArgs"); - - @JIT - public static void checkArity(ThreadContext context, StaticScope scope, Object[] args, Object keywords, Block block, int req, int opt, boolean rest, int keyrest) { - IRRuntimeHelpers.checkArity(context, scope, args, keywords, req, opt, rest, keyrest, block); - } - - @JIT - public static void checkAritySpecificArgs(ThreadContext context, StaticScope scope, Object[] args, Block block, int req, int opt, boolean rest, int keyrest) { - IRRuntimeHelpers.checkAritySpecificArgs(context, scope, args, req, opt, rest, keyrest, block); - } - - public static CallSite array(Lookup lookup, String name, MethodType type) { - MethodHandle handle = Binder - .from(type) - .collect(1, IRubyObject[].class) - .invoke(ARRAY_HANDLE); - - return new ConstantCallSite(handle); - } - - private static final MethodHandle ARRAY_HANDLE = - Binder - .from(RubyArray.class, ThreadContext.class, IRubyObject[].class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "array"); - - public static CallSite hash(Lookup lookup, String name, MethodType type) { - MethodHandle handle; - - int parameterCount = type.parameterCount(); - if (parameterCount == 1) { - handle = Binder - .from(lookup, type) - .cast(type.changeReturnType(RubyHash.class)) - .filter(0, RUNTIME_FROM_CONTEXT_HANDLE) - .invokeStaticQuiet(lookup, RubyHash.class, "newHash"); - } else if (!type.parameterType(parameterCount - 1).isArray() - && (parameterCount - 1) / 2 <= Helpers.MAX_SPECIFIC_ARITY_HASH) { - handle = Binder - .from(lookup, type) - .cast(type.changeReturnType(RubyHash.class)) - .filter(0, RUNTIME_FROM_CONTEXT_HANDLE) - .invokeStaticQuiet(lookup, Helpers.class, "constructSmallHash"); - } else { - handle = Binder - .from(lookup, type) - .collect(1, IRubyObject[].class) - .invoke(HASH_HANDLE); - } - - return new ConstantCallSite(handle); - } - - private static final MethodHandle HASH_HANDLE = - Binder - .from(RubyHash.class, ThreadContext.class, IRubyObject[].class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "hash"); - - public static CallSite kwargsHash(Lookup lookup, String name, MethodType type) { - MethodHandle handle = Binder - .from(lookup, type) - .collect(2, IRubyObject[].class) - .invoke(KWARGS_HASH_HANDLE); - - return new ConstantCallSite(handle); - } - - private static final MethodHandle KWARGS_HASH_HANDLE = - Binder - .from(RubyHash.class, ThreadContext.class, RubyHash.class, IRubyObject[].class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "kwargsHash"); - - public static Handle array() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "array", - sig(CallSite.class, Lookup.class, String.class, MethodType.class), - false); - } - - public static Handle hash() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "hash", - sig(CallSite.class, Lookup.class, String.class, MethodType.class), - false); - } - - public static Handle kwargsHash() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "kwargsHash", - sig(CallSite.class, Lookup.class, String.class, MethodType.class), - false); - } - - public static Handle invokeSuper() { - return SuperInvokeSite.BOOTSTRAP; - } - - public static Handle global() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "globalBootstrap", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, int.class), - false); - } - - public static RubyArray array(ThreadContext context, IRubyObject[] ary) { - assert ary.length > RubyArraySpecialized.MAX_PACKED_SIZE; - // Bootstrap.array() only dispatches here if ^^ holds - return RubyArray.newArrayNoCopy(context.runtime, ary); - } - - // We use LOOKUP here to have a full-featured MethodHandles.Lookup, avoiding jruby/jruby#7911 - private static final MethodHandle RUNTIME_FROM_CONTEXT_HANDLE = - Binder - .from(LOOKUP, Ruby.class, ThreadContext.class) - .getFieldQuiet("runtime"); - - public static RubyHash hash(ThreadContext context, IRubyObject[] pairs) { - Ruby runtime = context.runtime; - RubyHash hash = new RubyHash(runtime, pairs.length / 2 + 1); - for (int i = 0; i < pairs.length;) { - hash.fastASetCheckString(runtime, pairs[i++], pairs[i++]); - } - return hash; - } - - public static RubyHash kwargsHash(ThreadContext context, RubyHash hash, IRubyObject[] pairs) { - return IRRuntimeHelpers.dupKwargsHashAndPopulateFromArray(context, hash, pairs); - } - - static MethodHandle buildIndyHandle(InvokeSite site, CacheEntry entry) { - MethodHandle mh = null; - Signature siteToDyncall = site.signature.insertArgs(site.argOffset, arrayOf("class", "name"), arrayOf(RubyModule.class, String.class)); - DynamicMethod method = entry.method; - - if (method instanceof HandleMethod) { - HandleMethod handleMethod = (HandleMethod)method; - boolean blockGiven = site.signature.lastArgType() == Block.class; - - if (site.arity >= 0) { - mh = handleMethod.getHandle(site.arity); - if (mh != null) { - if (!blockGiven) mh = insertArguments(mh, mh.type().parameterCount() - 1, Block.NULL_BLOCK); - if (!site.functional) mh = dropArguments(mh, 1, IRubyObject.class); - } else { - mh = handleMethod.getHandle(-1); - if (!site.functional) mh = dropArguments(mh, 1, IRubyObject.class); - if (site.arity == 0) { - if (!blockGiven) { - mh = insertArguments(mh, mh.type().parameterCount() - 2, IRubyObject.NULL_ARRAY, Block.NULL_BLOCK); - } else { - mh = insertArguments(mh, mh.type().parameterCount() - 2, (Object)IRubyObject.NULL_ARRAY); - } - } else { - // bundle up varargs - if (!blockGiven) mh = insertArguments(mh, mh.type().parameterCount() - 1, Block.NULL_BLOCK); - - mh = SmartBinder.from(lookup(), siteToDyncall) - .collect("args", "arg.*", Helpers.constructObjectArrayHandle(site.arity)) - .invoke(mh) - .handle(); - } - } - } else { - mh = handleMethod.getHandle(-1); - if (mh != null) { - if (!site.functional) mh = dropArguments(mh, 1, IRubyObject.class); - if (!blockGiven) mh = insertArguments(mh, mh.type().parameterCount() - 1, Block.NULL_BLOCK); - - mh = SmartBinder.from(lookup(), siteToDyncall) - .invoke(mh) - .handle(); - } - } - - if (mh != null) { - mh = insertArguments(mh, site.argOffset, entry.sourceModule, site.name()); - - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tbound directly to handle " + Bootstrap.logMethod(method)); - } - } - } - - return mh; - } - - static MethodHandle buildGenericHandle(InvokeSite site, CacheEntry entry) { - SmartBinder binder; - DynamicMethod method = entry.method; - - binder = SmartBinder.from(site.signature); - - binder = permuteForGenericCall(binder, method, GENERIC_CALL_PERMUTE); - - binder = binder - .insert(2, new String[]{"rubyClass", "name"}, new Class[]{RubyModule.class, String.class}, entry.sourceModule, site.name()) - .insert(0, "method", DynamicMethod.class, method); - - if (site.arity > 3) { - binder = binder.collect("args", "arg.*", constructObjectArrayHandle(site.arity)); - } - - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tbound indirectly " + method + ", " + Bootstrap.logMethod(method)); - } - - return binder.invokeVirtualQuiet(LOOKUP, "call").handle(); - } - - private static SmartBinder permuteForGenericCall(SmartBinder binder, DynamicMethod method, String... basePermutes) { - if (methodWantsBlock(method)) { - binder = binder.permute(arrayOf(basePermutes, "block", String[]::new)); - } else { - binder = binder.permute(basePermutes); - } - return binder; - } - - private static boolean methodWantsBlock(DynamicMethod method) { - // only include block if native signature receives block, whatever its arity - boolean wantsBlock = true; - if (method instanceof NativeCallMethod) { - DynamicMethod.NativeCall nativeCall = ((NativeCallMethod) method).getNativeCall(); - // if it is a non-JI native call and does not want block, drop it - // JI calls may lazily convert blocks to an interface type (jruby/jruby#7246) - if (nativeCall != null && !nativeCall.isJava()) { - Class[] nativeSignature = nativeCall.getNativeSignature(); - - // no args or last arg not a block, do no pass block - if (nativeSignature.length == 0 || nativeSignature[nativeSignature.length - 1] != Block.class) { - wantsBlock = false; - } - } - } - return wantsBlock; - } - - static MethodHandle buildMethodMissingHandle(InvokeSite site, CacheEntry entry, IRubyObject self) { - SmartBinder binder; - DynamicMethod method = entry.method; - - if (site.arity >= 0) { - binder = SmartBinder.from(site.signature); - - binder = permuteForGenericCall(binder, method, GENERIC_CALL_PERMUTE) - .insert(2, - new String[]{"rubyClass", "name", "argName"} - , new Class[]{RubyModule.class, String.class, IRubyObject.class}, - entry.sourceModule, - site.name(), - self.getRuntime().newSymbol(site.methodName)) - .insert(0, "method", DynamicMethod.class, method) - .collect("args", "arg.*", Helpers.constructObjectArrayHandle(site.arity + 1)); - } else { - SmartHandle fold = SmartBinder.from( - site.signature - .permute("context", "self", "args", "block") - .changeReturn(IRubyObject[].class)) - .permute("args") - .insert(0, "argName", IRubyObject.class, self.getRuntime().newSymbol(site.methodName)) - .invokeStaticQuiet(LOOKUP, Helpers.class, "arrayOf"); - - binder = SmartBinder.from(site.signature); - - binder = permuteForGenericCall(binder, method, "context", "self", "args") - .fold("args2", fold); - binder = permuteForGenericCall(binder, method, "context", "self", "args2") - .insert(2, - new String[]{"rubyClass", "name"} - , new Class[]{RubyModule.class, String.class}, - entry.sourceModule, - site.name()) - .insert(0, "method", DynamicMethod.class, method); - } - - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tbound to method_missing for " + method + ", " + Bootstrap.logMethod(method)); - } - - return binder.invokeVirtualQuiet(LOOKUP, "call").handle(); - } - - static MethodHandle buildAttrHandle(InvokeSite site, CacheEntry entry, IRubyObject self) { - DynamicMethod method = entry.method; - - if (method instanceof AttrReaderMethod && site.arity == 0) { - AttrReaderMethod attrReader = (AttrReaderMethod) method; - String varName = attrReader.getVariableName(); - - // we getVariableAccessorForWrite here so it is eagerly created and we don't cache the DUMMY - VariableAccessor accessor = self.getType().getVariableAccessorForWrite(varName); - - // Ruby to attr reader - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - if (accessor instanceof FieldVariableAccessor) { - LOG.info(site.name() + "\tbound as field attr reader " + logMethod(method) + ":" + ((AttrReaderMethod)method).getVariableName()); - } else { - LOG.info(site.name() + "\tbound as attr reader " + logMethod(method) + ":" + ((AttrReaderMethod)method).getVariableName()); - } - } - - return createAttrReaderHandle(site, self, self.getType(), accessor); - } else if (method instanceof AttrWriterMethod && site.arity == 1) { - AttrWriterMethod attrReader = (AttrWriterMethod)method; - String varName = attrReader.getVariableName(); - - // we getVariableAccessorForWrite here so it is eagerly created and we don't cache the DUMMY - VariableAccessor accessor = self.getType().getVariableAccessorForWrite(varName); - - // Ruby to attr reader - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - if (accessor instanceof FieldVariableAccessor) { - LOG.info(site.name() + "\tbound as field attr writer " + logMethod(method) + ":" + ((AttrWriterMethod) method).getVariableName()); - } else { - LOG.info(site.name() + "\tbound as attr writer " + logMethod(method) + ":" + ((AttrWriterMethod) method).getVariableName()); - } - } - - return createAttrWriterHandle(site, self, self.getType(), accessor); - } - - return null; - } - - private static MethodHandle createAttrReaderHandle(InvokeSite site, IRubyObject self, RubyClass cls, VariableAccessor accessor) { - MethodHandle nativeTarget; - - MethodHandle filter = cls.getClassRuntime().getNullToNilHandle(); - - MethodHandle getValue; - - if (accessor instanceof FieldVariableAccessor) { - MethodHandle getter = ((FieldVariableAccessor)accessor).getGetter(); - getValue = Binder.from(site.type()) - .drop(0, site.argOffset - 1) - .filterReturn(filter) - .cast(methodType(Object.class, self.getClass())) - .invoke(getter); - } else { - getValue = Binder.from(site.type()) - .drop(0, site.argOffset - 1) - .filterReturn(filter) - .cast(methodType(Object.class, Object.class)) - .prepend(accessor) - .invokeVirtualQuiet(LOOKUP, "get"); - } - - // NOTE: Must not cache the fully-bound handle in the method, since it's specific to this class - - return getValue; - } - - public static IRubyObject valueOrNil(IRubyObject value, IRubyObject nil) { - return value == null ? nil : value; - } - - private static MethodHandle createAttrWriterHandle(InvokeSite site, IRubyObject self, RubyClass cls, VariableAccessor accessor) { - MethodHandle nativeTarget; - - MethodHandle filter = Binder - .from(IRubyObject.class, Object.class) - .drop(0) - .constant(cls.getRuntime().getNil()); - - MethodHandle setValue; - - if (accessor instanceof FieldVariableAccessor) { - MethodHandle setter = ((FieldVariableAccessor)accessor).getSetter(); - setValue = Binder.from(site.type()) - .drop(0, site.argOffset - 1) - .filterReturn(filter) - .cast(methodType(void.class, self.getClass(), Object.class)) - .invoke(setter); - } else { - setValue = Binder.from(site.type()) - .drop(0, site.argOffset - 1) - .filterReturn(filter) - .cast(methodType(void.class, Object.class, Object.class)) - .prepend(accessor) - .invokeVirtualQuiet(LOOKUP, "set"); - } - - return setValue; - } - - static MethodHandle buildJittedHandle(InvokeSite site, CacheEntry entry, boolean blockGiven) { - MethodHandle mh = null; - SmartBinder binder; - CompiledIRMethod compiledIRMethod = null; - DynamicMethod method = entry.method; - RubyModule sourceModule = entry.sourceModule; - - if (method instanceof CompiledIRMethod) { - compiledIRMethod = (CompiledIRMethod)method; - } else if (method instanceof MixedModeIRMethod) { - DynamicMethod actualMethod = ((MixedModeIRMethod)method).getActualMethod(); - if (actualMethod instanceof CompiledIRMethod) { - compiledIRMethod = (CompiledIRMethod) actualMethod; - } else { - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tfailed direct binding due to unjitted method " + Bootstrap.logMethod(method)); - } - } - } - - if (compiledIRMethod != null) { - - // attempt IR direct binding - // TODO: this will have to expand when we start specializing arities - - binder = SmartBinder.from(site.signature) - .permute("context", "self", "arg.*", "block"); - - if (site.arity == -1) { - // already [], nothing to do - mh = (MethodHandle)compiledIRMethod.getHandle(); - } else if (site.arity == 0) { - MethodHandle specific; - if ((specific = compiledIRMethod.getHandleFor(site.arity)) != null) { - mh = specific; - } else { - mh = (MethodHandle)compiledIRMethod.getHandle(); - binder = binder.insert(2, "args", IRubyObject.NULL_ARRAY); - } - } else { - MethodHandle specific; - if ((specific = compiledIRMethod.getHandleFor(site.arity)) != null) { - mh = specific; - } else { - mh = (MethodHandle) compiledIRMethod.getHandle(); - binder = binder.collect("args", "arg.*", Helpers.constructObjectArrayHandle(site.arity)); - } - } - - if (!blockGiven) { - binder = binder.append("block", Block.class, Block.NULL_BLOCK); - } - - binder = binder - .insert(1, "scope", StaticScope.class, compiledIRMethod.getStaticScope()) - .append("class", RubyModule.class, sourceModule) - .append("frameName", String.class, site.name()); - - mh = binder.invoke(mh).handle(); - - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tbound directly to jitted method " + Bootstrap.logMethod(method)); - } - } - - return mh; - } - - static MethodHandle buildNativeHandle(InvokeSite site, CacheEntry entry, boolean blockGiven) { - MethodHandle mh = null; - SmartBinder binder = null; - DynamicMethod method = entry.method; - - if (method instanceof NativeCallMethod && ((NativeCallMethod) method).getNativeCall() != null) { - NativeCallMethod nativeMethod = (NativeCallMethod)method; - DynamicMethod.NativeCall nativeCall = nativeMethod.getNativeCall(); - - DynamicMethod.NativeCall nc = nativeCall; - - if (nc.isJava()) { - return JavaBootstrap.createJavaHandle(site, method); - } else { - int nativeArgCount = getNativeArgCount(method, nativeCall); - - if (nativeArgCount >= 0) { // native methods only support arity 3 - if (nativeArgCount == site.arity) { - // nothing to do - binder = SmartBinder.from(lookup(), site.signature); - } else { - // arity mismatch...leave null and use DynamicMethod.call below - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tdid not match the primary arity for a native method " + Bootstrap.logMethod(method)); - } - } - } else { - // varargs - if (site.arity == -1) { - // ok, already passing [] - binder = SmartBinder.from(lookup(), site.signature); - } else if (site.arity == 0) { - // no args, insert dummy - binder = SmartBinder.from(lookup(), site.signature) - .insert(site.argOffset, "args", IRubyObject.NULL_ARRAY); - } else { - // 1 or more args, collect into [] - binder = SmartBinder.from(lookup(), site.signature) - .collect("args", "arg.*", Helpers.constructObjectArrayHandle(site.arity)); - } - } - - if (binder != null) { - - // clean up non-arguments, ordering, types - if (!nc.hasContext()) { - binder = binder.drop("context"); - } - - if (nc.hasBlock() && !blockGiven) { - binder = binder.append("block", Block.NULL_BLOCK); - } else if (!nc.hasBlock() && blockGiven) { - binder = binder.drop("block"); - } - - if (nc.isStatic()) { - mh = binder - .permute("context", "self", "arg.*", "block") // filter caller - .cast(nc.getNativeReturn(), nc.getNativeSignature()) - .invokeStaticQuiet(LOOKUP, nc.getNativeTarget(), nc.getNativeName()) - .handle(); - } else { - mh = binder - .permute("self", "context", "arg.*", "block") // filter caller, move self - .castArg("self", nc.getNativeTarget()) - .castVirtual(nc.getNativeReturn(), nc.getNativeTarget(), nc.getNativeSignature()) - .invokeVirtualQuiet(LOOKUP, nc.getNativeName()) - .handle(); - } - - if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { - LOG.info(site.name() + "\tbound directly to JVM method " + Bootstrap.logMethod(method)); - } - - JRubyMethod anno = nativeCall.getMethod().getAnnotation(JRubyMethod.class); - if (anno != null && anno.frame()) { - mh = InvocationLinker.wrapWithFrameOnly(site.signature, entry.sourceModule, site.name(), mh); - } - } - } - } - - return mh; - } - - public static int getNativeArgCount(DynamicMethod method, DynamicMethod.NativeCall nativeCall) { - // if non-Java, must: - // * exactly match arities or both are [] boxed - // * 3 or fewer arguments - return getArgCount(nativeCall.getNativeSignature(), nativeCall.isStatic()); - } - - private static int getArgCount(Class[] args, boolean isStatic) { - int length = args.length; - boolean hasContext = false; - if (isStatic) { - if (args.length > 1 && args[0] == ThreadContext.class) { - length--; - hasContext = true; - } - - // remove self object - assert args.length >= 1; - length--; - - if (args.length > 1 && args[args.length - 1] == Block.class) { - length--; - } - - if (length == 1) { - if (hasContext && args[2] == IRubyObject[].class) { - length = -1; - } else if (args[1] == IRubyObject[].class) { - length = -1; - } - } - } else { - if (args.length > 0 && args[0] == ThreadContext.class) { - length--; - hasContext = true; - } - - if (args.length > 0 && args[args.length - 1] == Block.class) { - length--; - } - - if (length == 1) { - if (hasContext && args[1] == IRubyObject[].class) { - length = -1; - } else if (args[0] == IRubyObject[].class) { - length = -1; - } - } - } - return length; - } - - public static boolean testType(RubyClass original, IRubyObject self) { - // naive test - return original == RubyBasicObject.getMetaClass(self); - } - /////////////////////////////////////////////////////////////////////////// // Fixnum binding @@ -1048,81 +292,6 @@ public static CallSite getHeapLocalOrNilBootstrap(Lookup lookup, String name, Me return site; } - public static CallSite globalBootstrap(Lookup lookup, String name, MethodType type, String file, int line) throws Throwable { - String[] names = name.split(":"); - String operation = names[0]; - String varName = JavaNameMangler.demangleMethodName(names[1]); - GlobalSite site = new GlobalSite(type, varName, file, line); - MethodHandle handle; - - if (operation.equals("get")) { - handle = lookup.findStatic(Bootstrap.class, "getGlobalFallback", methodType(IRubyObject.class, GlobalSite.class, ThreadContext.class)); - } else { - handle = lookup.findStatic(Bootstrap.class, "setGlobalFallback", methodType(void.class, GlobalSite.class, IRubyObject.class, ThreadContext.class)); - } - - handle = handle.bindTo(site); - site.setTarget(handle); - - return site; - } - - public static IRubyObject getGlobalFallback(GlobalSite site, ThreadContext context) throws Throwable { - Ruby runtime = context.runtime; - GlobalVariable variable = runtime.getGlobalVariables().getVariable(site.name()); - - if (site.failures() > Options.INVOKEDYNAMIC_GLOBAL_MAXFAIL.load() || - variable.getScope() != GlobalVariable.Scope.GLOBAL || - RubyGlobal.UNCACHED_GLOBALS.contains(site.name())) { - - // use uncached logic forever - if (Options.INVOKEDYNAMIC_LOG_GLOBALS.load()) LOG.info("global " + site.name() + " (" + site.file() + ":" + site.line() + ") uncacheable or rebound > " + Options.INVOKEDYNAMIC_GLOBAL_MAXFAIL.load() + " times, reverting to simple lookup"); - - MethodHandle uncached = lookup().findStatic(Bootstrap.class, "getGlobalUncached", methodType(IRubyObject.class, GlobalVariable.class)); - uncached = uncached.bindTo(variable); - uncached = dropArguments(uncached, 0, ThreadContext.class); - site.setTarget(uncached); - return (IRubyObject)uncached.invokeWithArguments(context); - } - - Invalidator invalidator = variable.getInvalidator(); - IRubyObject value = variable.getAccessor().getValue(); - - MethodHandle target = constant(IRubyObject.class, value); - target = dropArguments(target, 0, ThreadContext.class); - MethodHandle fallback = lookup().findStatic(Bootstrap.class, "getGlobalFallback", methodType(IRubyObject.class, GlobalSite.class, ThreadContext.class)); - fallback = fallback.bindTo(site); - - target = ((SwitchPoint)invalidator.getData()).guardWithTest(target, fallback); - - site.setTarget(target); - -// if (Options.INVOKEDYNAMIC_LOG_GLOBALS.load()) LOG.info("global " + site.name() + " (" + site.file() + ":" + site.line() + ") cached"); - - return value; - } - - public static IRubyObject getGlobalUncached(GlobalVariable variable) throws Throwable { - return variable.getAccessor().getValue(); - } - - public static void setGlobalFallback(GlobalSite site, IRubyObject value, ThreadContext context) throws Throwable { - Ruby runtime = context.runtime; - GlobalVariable variable = runtime.getGlobalVariables().getVariable(site.name()); - MethodHandle uncached = lookup().findStatic(Bootstrap.class, "setGlobalUncached", methodType(void.class, GlobalVariable.class, IRubyObject.class)); - uncached = uncached.bindTo(variable); - uncached = dropArguments(uncached, 1, ThreadContext.class); - site.setTarget(uncached); - uncached.invokeWithArguments(value, context); - } - - public static void setGlobalUncached(GlobalVariable variable, IRubyObject value) throws Throwable { - // FIXME: duplicated logic from GlobalVariables.set - variable.getAccessor().setValue(value); - variable.trace(value); - variable.invalidate(); - } - public static Handle prepareBlock() { return new Handle( Opcodes.H_INVOKESTATIC, @@ -1179,17 +348,6 @@ public static CallSite prepareBlock(Lookup lookup, String name, MethodType type, return new ConstantCallSite(blockMaker); } - public CallSite checkArrayArity(Lookup lookup, String name, MethodType methodType, int required, int opt, int rest) { - return new ConstantCallSite(MethodHandles.insertArguments(CHECK_ARRAY_ARITY, 2, required, opt, (rest == 0 ? false : true))); - } - - public static final Handle CHECK_ARRAY_ARITY_BOOTSTRAP = new Handle(Opcodes.H_INVOKESTATIC, p(Helpers.class), "checkArrayArity", sig(CallSite.class, Lookup.class, String.class, MethodType.class, int.class, int.class, int.class), false); - - private static final MethodHandle CHECK_ARRAY_ARITY = - Binder - .from(void.class, ThreadContext.class, RubyArray.class, int.class, int.class, boolean.class) - .invokeStaticQuiet(LOOKUP, Helpers.class, "irCheckArgsArrayArity"); - static String logMethod(DynamicMethod method) { return "[#" + method.getSerialNumber() + " " + method.getImplementationClass().getMethodLocation() + "]"; } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/CheckArityBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/CheckArityBootstrap.java new file mode 100644 index 00000000000..6f80c7a60ad --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/CheckArityBootstrap.java @@ -0,0 +1,76 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.RubyArray; +import org.jruby.ir.JIT; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.parser.StaticScope; +import org.jruby.runtime.Block; +import org.jruby.runtime.Helpers; +import org.jruby.runtime.ThreadContext; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static java.lang.invoke.MethodHandles.insertArguments; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class CheckArityBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + public static final Handle CHECK_ARITY_SPECIFIC_ARGS = new Handle( + Opcodes.H_INVOKESTATIC, + p(CheckArityBootstrap.class), + "checkAritySpecificArgs", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, int.class, int.class), + false); + public static final Handle CHECK_ARITY = new Handle( + Opcodes.H_INVOKESTATIC, + p(CheckArityBootstrap.class), + "checkArity", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, int.class, int.class), + false); + public static final Handle CHECK_ARRAY_ARITY_BOOTSTRAP = new Handle(Opcodes.H_INVOKESTATIC, p(Helpers.class), "checkArrayArity", sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, int.class), false); + private static final MethodHandle CHECK_ARITY_HANDLE = + Binder + .from(void.class, ThreadContext.class, StaticScope.class, Object[].class, Object.class, Block.class, int.class, int.class, boolean.class, int.class) + .invokeStaticQuiet(LOOKUP, CheckArityBootstrap.class, "checkArity"); + private static final MethodHandle CHECK_ARITY_SPECIFIC_ARGS_HANDLE = + Binder + .from(void.class, ThreadContext.class, StaticScope.class, Object[].class, Block.class, int.class, int.class, boolean.class, int.class) + .invokeStaticQuiet(LOOKUP, CheckArityBootstrap.class, "checkAritySpecificArgs"); + private static final MethodHandle CHECK_ARRAY_ARITY = + Binder + .from(void.class, ThreadContext.class, RubyArray.class, int.class, int.class, boolean.class) + .invokeStaticQuiet(LOOKUP, Helpers.class, "irCheckArgsArrayArity"); + + @JIT + public static CallSite checkArity(MethodHandles.Lookup lookup, String name, MethodType type, int req, int opt, int rest, int keyrest) { + return new ConstantCallSite(insertArguments(CHECK_ARITY_HANDLE, 5, req, opt, rest == 0 ? false : true, keyrest)); + } + + @JIT + public static CallSite checkAritySpecificArgs(MethodHandles.Lookup lookup, String name, MethodType type, int req, int opt, int rest, int keyrest) { + return new ConstantCallSite(insertArguments(CHECK_ARITY_SPECIFIC_ARGS_HANDLE, 4, req, opt, rest == 0 ? false : true, keyrest)); + } + + @JIT + public static void checkArity(ThreadContext context, StaticScope scope, Object[] args, Object keywords, Block block, int req, int opt, boolean rest, int keyrest) { + IRRuntimeHelpers.checkArity(context, scope, args, keywords, req, opt, rest, keyrest, block); + } + + @JIT + public static void checkAritySpecificArgs(ThreadContext context, StaticScope scope, Object[] args, Block block, int req, int opt, boolean rest, int keyrest) { + IRRuntimeHelpers.checkAritySpecificArgs(context, scope, args, req, opt, rest, keyrest, block); + } + + public static CallSite checkArrayArity(MethodHandles.Lookup lookup, String name, MethodType methodType, int required, int opt, int rest) { + return new ConstantCallSite(MethodHandles.insertArguments(CHECK_ARRAY_ARITY, 2, required, opt, (rest == 0 ? false : true))); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/ConstantLookupSite.java b/core/src/main/java/org/jruby/ir/targets/indy/ConstantLookupSite.java index 32fdae1b5e7..5fb91edc95f 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/ConstantLookupSite.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/ConstantLookupSite.java @@ -40,6 +40,8 @@ public class ConstantLookupSite extends MutableCallSite { private final SiteTracker tracker = new SiteTracker(); + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + public static final Handle BOOTSTRAP = new Handle( Opcodes.H_INVOKESTATIC, p(ConstantLookupSite.class), @@ -105,7 +107,7 @@ public IRubyObject searchConst(ThreadContext context, StaticScope staticScope) { .constant(constant); MethodHandle fallback = Binder.from(type()) .insert(0, this) - .invokeVirtualQuiet(Bootstrap.LOOKUP, "searchConst"); + .invokeVirtualQuiet(LOOKUP, "searchConst"); setTarget(switchPoint.guardWithTest(target, fallback)); @@ -296,7 +298,7 @@ public IRubyObject lexicalSearchConst(ThreadContext context, StaticScope scope) MethodHandle fallback = Binder.from(type()) .insert(0, this) - .invokeVirtualQuiet(Bootstrap.LOOKUP, "lexicalSearchConst"); + .invokeVirtualQuiet(LOOKUP, "lexicalSearchConst"); setTarget(switchPoint.guardWithTest(target, fallback)); @@ -312,7 +314,7 @@ private MethodHandle SMFC() { if (_SMFC != null) return _SMFC; return _SMFC = Binder.from(type()) .insert(0, this) - .invokeVirtualQuiet(Bootstrap.LOOKUP, "searchModuleForConst"); + .invokeVirtualQuiet(LOOKUP, "searchModuleForConst"); } private MethodHandle _noCacheSMFC; @@ -320,7 +322,7 @@ private MethodHandle noCacheSMFC() { if (_noCacheSMFC != null) return _noCacheSMFC; return _noCacheSMFC = Binder.from(type()) .insert(0, this) - .invokeVirtualQuiet(Bootstrap.LOOKUP, "noCacheSearchModuleForConst"); + .invokeVirtualQuiet(LOOKUP, "noCacheSearchModuleForConst"); } private MethodHandle _ISC; @@ -328,7 +330,7 @@ private MethodHandle ISC() { if (_ISC != null) return _ISC; return _ISC = Binder.from(type()) .insert(0, this) - .invokeVirtualQuiet(Bootstrap.LOOKUP, "inheritanceSearchConst"); + .invokeVirtualQuiet(LOOKUP, "inheritanceSearchConst"); } private MethodHandle _noCacheISC; @@ -336,6 +338,6 @@ private MethodHandle noCacheISC() { if (_noCacheISC != null) return _noCacheISC; return _noCacheISC = Binder.from(type()) .insert(0, this) - .invokeVirtualQuiet(Bootstrap.LOOKUP, "noCacheInheritanceSearchConst"); + .invokeVirtualQuiet(LOOKUP, "noCacheInheritanceSearchConst"); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/HashBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/HashBootstrap.java new file mode 100644 index 00000000000..5c755a22c99 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/HashBootstrap.java @@ -0,0 +1,99 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.Ruby; +import org.jruby.RubyHash; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.runtime.Helpers; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class HashBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + private static final MethodHandle HASH_HANDLE = + Binder + .from(RubyHash.class, ThreadContext.class, IRubyObject[].class) + .invokeStaticQuiet(LOOKUP, HashBootstrap.class, "hash"); + private static final MethodHandle KWARGS_HASH_HANDLE = + Binder + .from(RubyHash.class, ThreadContext.class, RubyHash.class, IRubyObject[].class) + .invokeStaticQuiet(LOOKUP, HashBootstrap.class, "kwargsHash"); + public static final Handle HASH_H = new Handle( + Opcodes.H_INVOKESTATIC, + p(HashBootstrap.class), + "hash", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class), + false); + public static final Handle KWARGS_HASH_H = new Handle( + Opcodes.H_INVOKESTATIC, + p(HashBootstrap.class), + "kwargsHash", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class), + false); + // We use LOOKUP here to have a full-featured MethodHandles.Lookup, avoiding jruby/jruby#7911 + private static final MethodHandle RUNTIME_FROM_CONTEXT_HANDLE = + Binder + .from(LOOKUP, Ruby.class, ThreadContext.class) + .getFieldQuiet("runtime"); + + public static CallSite hash(MethodHandles.Lookup lookup, String name, MethodType type) { + MethodHandle handle; + + int parameterCount = type.parameterCount(); + if (parameterCount == 1) { + handle = Binder + .from(lookup, type) + .cast(type.changeReturnType(RubyHash.class)) + .filter(0, RUNTIME_FROM_CONTEXT_HANDLE) + .invokeStaticQuiet(lookup, RubyHash.class, "newHash"); + } else if (!type.parameterType(parameterCount - 1).isArray() + && (parameterCount - 1) / 2 <= Helpers.MAX_SPECIFIC_ARITY_HASH) { + handle = Binder + .from(lookup, type) + .cast(type.changeReturnType(RubyHash.class)) + .filter(0, RUNTIME_FROM_CONTEXT_HANDLE) + .invokeStaticQuiet(lookup, Helpers.class, "constructSmallHash"); + } else { + handle = Binder + .from(lookup, type) + .collect(1, IRubyObject[].class) + .invoke(HASH_HANDLE); + } + + return new ConstantCallSite(handle); + } + + public static CallSite kwargsHash(MethodHandles.Lookup lookup, String name, MethodType type) { + MethodHandle handle = Binder + .from(lookup, type) + .collect(2, IRubyObject[].class) + .invoke(KWARGS_HASH_HANDLE); + + return new ConstantCallSite(handle); + } + + public static RubyHash hash(ThreadContext context, IRubyObject[] pairs) { + Ruby runtime = context.runtime; + RubyHash hash = new RubyHash(runtime, pairs.length / 2 + 1); + for (int i = 0; i < pairs.length;) { + hash.fastASetCheckString(runtime, pairs[i++], pairs[i++]); + } + return hash; + } + + public static RubyHash kwargsHash(ThreadContext context, RubyHash hash, IRubyObject[] pairs) { + return IRRuntimeHelpers.dupKwargsHashAndPopulateFromArray(context, hash, pairs); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyArgumentsCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyArgumentsCompiler.java index eeab3596aa9..cc3aba45a24 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyArgumentsCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyArgumentsCompiler.java @@ -23,6 +23,6 @@ public void kwargsHash(int length) { if (length > IRBytecodeAdapter.MAX_ARGUMENTS / 2) throw new NotCompilableException("kwargs hash has more than " + (IRBytecodeAdapter.MAX_ARGUMENTS / 2) + " pairs"); - compiler.adapter.invokedynamic("kwargsHash", CodegenUtils.sig(JVM.OBJECT, params(ThreadContext.class, RubyHash.class, JVM.OBJECT, length * 2)), Bootstrap.kwargsHash()); + compiler.adapter.invokedynamic("kwargsHash", CodegenUtils.sig(JVM.OBJECT, params(ThreadContext.class, RubyHash.class, JVM.OBJECT, length * 2)), HashBootstrap.KWARGS_HASH_H); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyBranchCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyBranchCompiler.java index e0b3c472da9..8f50d88e3d8 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyBranchCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyBranchCompiler.java @@ -23,13 +23,13 @@ public IndyBranchCompiler(IRBytecodeAdapter compiler) { @Override public void branchIfNil(Label label) { - compiler.adapter.invokedynamic("isNil", sig(boolean.class, IRubyObject.class), Bootstrap.isNilBoot()); + compiler.adapter.invokedynamic("isNil", sig(boolean.class, IRubyObject.class), IsNilSite.IS_NIL_BOOTSTRAP_HANDLE); compiler.adapter.iftrue(label); } @Override public void branchIfTruthy(Label label) { - compiler.adapter.invokedynamic("isTrue", sig(boolean.class, IRubyObject.class), Bootstrap.isTrueBoot()); + compiler.adapter.invokedynamic("isTrue", sig(boolean.class, IRubyObject.class), IsTrueSite.IS_TRUE_BOOTSTRAP_HANDLE); compiler.adapter.iftrue(label); } @@ -46,14 +46,14 @@ public void btrue(Label label) { public void checkArgsArity(Runnable args, int required, int opt, boolean rest) { compiler.loadContext(); args.run(); - compiler.adapter.invokedynamic("checkArrayArity", sig(void.class, ThreadContext.class, RubyArray.class), Bootstrap.CHECK_ARRAY_ARITY_BOOTSTRAP, required, opt, rest ? 1 : 0); + compiler.adapter.invokedynamic("checkArrayArity", sig(void.class, ThreadContext.class, RubyArray.class), CheckArityBootstrap.CHECK_ARRAY_ARITY_BOOTSTRAP, required, opt, rest ? 1 : 0); } public void checkArity(int required, int opt, boolean rest, int restKey) { compiler.adapter.invokedynamic( "checkArity", sig(void.class, ThreadContext.class, StaticScope.class, Object[].class, Object.class, Block.class), - Bootstrap.CHECK_ARITY, + CheckArityBootstrap.CHECK_ARITY, required, opt, rest ? 1 : 0, restKey); } @@ -61,7 +61,7 @@ public void checkAritySpecificArgs(int required, int opt, boolean rest, int rest compiler.adapter.invokedynamic( "checkArity", sig(void.class, ThreadContext.class, StaticScope.class, Object[].class, Block.class), - Bootstrap.CHECK_ARITY_SPECIFIC_ARGS, + CheckArityBootstrap.CHECK_ARITY_SPECIFIC_ARGS, required, opt, rest ? 1 : 0, restKey); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyDynamicValueCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyDynamicValueCompiler.java index 03df2b978bd..28cc0f2e48f 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyDynamicValueCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyDynamicValueCompiler.java @@ -37,13 +37,13 @@ public void array(int length) { return; } - compiler.adapter.invokedynamic("array", CodegenUtils.sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, length)), Bootstrap.array()); + compiler.adapter.invokedynamic("array", CodegenUtils.sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, length)), ArrayBootstrap.ARRAY_H); } public void hash(int length) { if (length > IRBytecodeAdapter.MAX_ARGUMENTS / 2) throw new NotCompilableException("literal hash has more than " + (IRBytecodeAdapter.MAX_ARGUMENTS / 2) + " pairs"); - compiler.adapter.invokedynamic("hash", sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, length * 2)), Bootstrap.hash()); + compiler.adapter.invokedynamic("hash", sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, length * 2)), HashBootstrap.HASH_H); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyGlobalVariableCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyGlobalVariableCompiler.java index 7ae0d457a68..6362ee36f5d 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyGlobalVariableCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyGlobalVariableCompiler.java @@ -4,6 +4,7 @@ import org.jruby.ir.targets.IRBytecodeAdapter; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.invokedynamic.GlobalSite; import org.jruby.util.JavaNameMangler; import static org.jruby.util.CodegenUtils.sig; @@ -21,7 +22,7 @@ public void getGlobalVariable(String name, String file) { compiler.adapter.invokedynamic( "get:" + JavaNameMangler.mangleMethodName(name), sig(IRubyObject.class, ThreadContext.class), - Bootstrap.global(), + org.jruby.runtime.invokedynamic.GlobalSite.GLOBAL_BOOTSTRAP_H, file, compiler.getLastLine()); } @@ -31,7 +32,7 @@ public void setGlobalVariable(String name, String file) { compiler.adapter.invokedynamic( "set:" + JavaNameMangler.mangleMethodName(name), sig(void.class, IRubyObject.class, ThreadContext.class), - Bootstrap.global(), + GlobalSite.GLOBAL_BOOTSTRAP_H, file, compiler.getLastLine()); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java index d28d80469d5..3b22b36698c 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java @@ -150,9 +150,9 @@ public void invokeInstanceSuper(String file, String name, int arity, boolean has String splatmapString = IRRuntimeHelpers.encodeSplatmap(splatmap); if (hasClosure) { String operation = literalClosure ? "invokeInstanceSuperIter" : "invokeInstanceSuper"; - compiler.adapter.invokedynamic(operation + ":" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic(operation + ":" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } else { - compiler.adapter.invokedynamic("invokeInstanceSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic("invokeInstanceSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } } @@ -163,9 +163,9 @@ public void invokeClassSuper(String file, String name, int arity, boolean hasClo String splatmapString = IRRuntimeHelpers.encodeSplatmap(splatmap); if (hasClosure) { String operation = literalClosure ? "invokeClassSuperIter" : "invokeClassSuper"; - compiler.adapter.invokedynamic(operation + ":" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic(operation + ":" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } else { - compiler.adapter.invokedynamic("invokeClassSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic("invokeClassSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } } @@ -176,9 +176,9 @@ public void invokeUnresolvedSuper(String file, String name, int arity, boolean h String splatmapString = IRRuntimeHelpers.encodeSplatmap(splatmap); if (hasClosure) { String operation = literalClosure ? "invokeUnresolvedSuperIter" : "invokeUnresolvedSuper"; - compiler.adapter.invokedynamic(operation + ":" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic(operation + ":" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } else { - compiler.adapter.invokedynamic("invokeUnresolvedSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic("invokeUnresolvedSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } } @@ -188,9 +188,9 @@ public void invokeZSuper(String file, String name, int arity, boolean hasClosure String splatmapString = IRRuntimeHelpers.encodeSplatmap(splatmap); if (hasClosure) { - compiler.adapter.invokedynamic("invokeZSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic("invokeZSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity, Block.class)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } else { - compiler.adapter.invokedynamic("invokeZSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), Bootstrap.invokeSuper(), splatmapString, flags, file, compiler.getLastLine()); + compiler.adapter.invokedynamic("invokeZSuper:" + JavaNameMangler.mangleMethodName(name), sig(JVM.OBJECT, params(ThreadContext.class, JVM.OBJECT, JVM.OBJECT, RubyClass.class, JVM.OBJECT, arity)), SuperInvokeSite.BOOTSTRAP, splatmapString, flags, file, compiler.getLastLine()); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/InvokeSite.java b/core/src/main/java/org/jruby/ir/targets/indy/InvokeSite.java index d4956c9cc8c..a081e72ce25 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/InvokeSite.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/InvokeSite.java @@ -15,8 +15,15 @@ import org.jruby.RubyNil; import org.jruby.RubyStruct; import org.jruby.RubySymbol; +import org.jruby.anno.JRubyMethod; import org.jruby.internal.runtime.methods.AliasMethod; +import org.jruby.internal.runtime.methods.AttrReaderMethod; +import org.jruby.internal.runtime.methods.AttrWriterMethod; +import org.jruby.internal.runtime.methods.CompiledIRMethod; import org.jruby.internal.runtime.methods.DynamicMethod; +import org.jruby.internal.runtime.methods.HandleMethod; +import org.jruby.internal.runtime.methods.MixedModeIRMethod; +import org.jruby.internal.runtime.methods.NativeCallMethod; import org.jruby.internal.runtime.methods.PartialDelegatingMethod; import org.jruby.ir.JIT; import org.jruby.ir.runtime.IRRuntimeHelpers; @@ -24,6 +31,7 @@ import org.jruby.java.invokers.InstanceFieldGetter; import org.jruby.java.invokers.InstanceFieldSetter; import org.jruby.javasupport.JavaUtil; +import org.jruby.parser.StaticScope; import org.jruby.runtime.Block; import org.jruby.runtime.CallType; import org.jruby.runtime.Helpers; @@ -32,7 +40,10 @@ import org.jruby.runtime.Visibility; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.runtime.callsite.CacheEntry; +import org.jruby.runtime.invokedynamic.InvocationLinker; import org.jruby.runtime.invokedynamic.JRubyCallSite; +import org.jruby.runtime.ivars.FieldVariableAccessor; +import org.jruby.runtime.ivars.VariableAccessor; import org.jruby.util.cli.Options; import org.jruby.util.log.Logger; import org.jruby.util.log.LoggerFactory; @@ -49,9 +60,13 @@ import java.util.Map; import static java.lang.invoke.MethodHandles.constant; +import static java.lang.invoke.MethodHandles.dropArguments; import static java.lang.invoke.MethodHandles.foldArguments; import static java.lang.invoke.MethodHandles.insertArguments; import static java.lang.invoke.MethodHandles.lookup; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.runtime.Helpers.arrayOf; +import static org.jruby.runtime.Helpers.constructObjectArrayHandle; import static org.jruby.runtime.invokedynamic.JRubyCallSite.SITE_ID; /** @@ -60,6 +75,8 @@ public abstract class InvokeSite extends MutableCallSite { private static final Logger LOG = LoggerFactory.getLogger(InvokeSite.class); + private static final String[] GENERIC_CALL_PERMUTE = {"context", "self", "arg.*"}; + static { // enable DEBUG output if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) LOG.setDebugEnable(true); } @@ -82,6 +99,469 @@ public abstract class InvokeSite extends MutableCallSite { private boolean literalClosure; protected CacheEntry cache = CacheEntry.NULL_CACHE; + public static boolean testType(RubyClass original, IRubyObject self) { + // naive test + return original == RubyBasicObject.getMetaClass(self); + } + + MethodHandle buildIndyHandle(CacheEntry entry) { + MethodHandle mh = null; + Signature siteToDyncall = signature.insertArgs(argOffset, arrayOf("class", "name"), arrayOf(RubyModule.class, String.class)); + DynamicMethod method = entry.method; + + if (method instanceof HandleMethod) { + HandleMethod handleMethod = (HandleMethod)method; + boolean blockGiven = signature.lastArgType() == Block.class; + + if (arity >= 0) { + mh = handleMethod.getHandle(arity); + if (mh != null) { + if (!blockGiven) mh = insertArguments(mh, mh.type().parameterCount() - 1, Block.NULL_BLOCK); + if (!functional) mh = dropArguments(mh, 1, IRubyObject.class); + } else { + mh = handleMethod.getHandle(-1); + if (!functional) mh = dropArguments(mh, 1, IRubyObject.class); + if (arity == 0) { + if (!blockGiven) { + mh = insertArguments(mh, mh.type().parameterCount() - 2, IRubyObject.NULL_ARRAY, Block.NULL_BLOCK); + } else { + mh = insertArguments(mh, mh.type().parameterCount() - 2, (Object)IRubyObject.NULL_ARRAY); + } + } else { + // bundle up varargs + if (!blockGiven) mh = insertArguments(mh, mh.type().parameterCount() - 1, Block.NULL_BLOCK); + + mh = SmartBinder.from(lookup(), siteToDyncall) + .collect("args", "arg.*", Helpers.constructObjectArrayHandle(arity)) + .invoke(mh) + .handle(); + } + } + } else { + mh = handleMethod.getHandle(-1); + if (mh != null) { + if (!functional) mh = dropArguments(mh, 1, IRubyObject.class); + if (!blockGiven) mh = insertArguments(mh, mh.type().parameterCount() - 1, Block.NULL_BLOCK); + + mh = SmartBinder.from(lookup(), siteToDyncall) + .invoke(mh) + .handle(); + } + } + + if (mh != null) { + mh = insertArguments(mh, argOffset, entry.sourceModule, name()); + + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(name() + "\tbound directly to handle " + Bootstrap.logMethod(method)); + } + } + } + + return mh; + } + + MethodHandle buildGenericHandle(CacheEntry entry) { + SmartBinder binder; + DynamicMethod method = entry.method; + + binder = SmartBinder.from(signature); + + binder = permuteForGenericCall(binder, method, GENERIC_CALL_PERMUTE); + + + binder = binder + .insert(2, new String[]{"rubyClass", "name"}, new Class[]{RubyModule.class, String.class}, entry.sourceModule, name()) + .insert(0, "method", DynamicMethod.class, method); + + if (arity > 3) { + binder = binder.collect("args", "arg.*", constructObjectArrayHandle(arity)); + } + + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(name() + "\tbound indirectly " + method + ", " + Bootstrap.logMethod(method)); + } + + return binder.invokeVirtualQuiet(LOOKUP, "call").handle(); + } + + private static SmartBinder permuteForGenericCall(SmartBinder binder, DynamicMethod method, String... basePermutes) { + if (methodWantsBlock(method)) { + binder = binder.permute(arrayOf(basePermutes, "block", String[]::new)); + } else { + binder = binder.permute(basePermutes); + } + return binder; + } + + private static boolean methodWantsBlock(DynamicMethod method) { + // only include block if native signature receives block, whatever its arity + boolean wantsBlock = true; + if (method instanceof NativeCallMethod) { + DynamicMethod.NativeCall nativeCall = ((NativeCallMethod) method).getNativeCall(); + // if it is a non-JI native call and does not want block, drop it + // JI calls may lazily convert blocks to an interface type (jruby/jruby#7246) + if (nativeCall != null && !nativeCall.isJava()) { + Class[] nativeSignature = nativeCall.getNativeSignature(); + + // no args or last arg not a block, do no pass block + if (nativeSignature.length == 0 || nativeSignature[nativeSignature.length - 1] != Block.class) { + wantsBlock = false; + } + } + } + return wantsBlock; + } + + static MethodHandle buildMethodMissingHandle(InvokeSite site, CacheEntry entry, IRubyObject self) { + SmartBinder binder; + DynamicMethod method = entry.method; + + if (site.arity >= 0) { + binder = SmartBinder.from(site.signature); + + binder = permuteForGenericCall(binder, method, GENERIC_CALL_PERMUTE) + .insert(2, + new String[]{"rubyClass", "name", "argName"} + , new Class[]{RubyModule.class, String.class, IRubyObject.class}, + entry.sourceModule, + site.name(), + self.getRuntime().newSymbol(site.methodName)) + .insert(0, "method", DynamicMethod.class, method) + .collect("args", "arg.*", Helpers.constructObjectArrayHandle(site.arity + 1)); + } else { + SmartHandle fold = SmartBinder.from( + site.signature + .permute("context", "self", "args", "block") + .changeReturn(IRubyObject[].class)) + .permute("args") + .insert(0, "argName", IRubyObject.class, self.getRuntime().newSymbol(site.methodName)) + .invokeStaticQuiet(LOOKUP, Helpers.class, "arrayOf"); + + binder = SmartBinder.from(site.signature); + + binder = permuteForGenericCall(binder, method, "context", "self", "args") + .fold("args2", fold); + binder = permuteForGenericCall(binder, method, "context", "self", "args2") + .insert(2, + new String[]{"rubyClass", "name"} + , new Class[]{RubyModule.class, String.class}, + entry.sourceModule, + site.name()) + .insert(0, "method", DynamicMethod.class, method); + } + + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(site.name() + "\tbound to method_missing for " + method + ", " + Bootstrap.logMethod(method)); + } + + return binder.invokeVirtualQuiet(LOOKUP, "call").handle(); + } + + MethodHandle buildAttrHandle(CacheEntry entry, IRubyObject self) { + DynamicMethod method = entry.method; + + if (method instanceof AttrReaderMethod && arity == 0) { + AttrReaderMethod attrReader = (AttrReaderMethod) method; + String varName = attrReader.getVariableName(); + + // we getVariableAccessorForWrite here so it is eagerly created and we don't cache the DUMMY + VariableAccessor accessor = self.getType().getVariableAccessorForWrite(varName); + + // Ruby to attr reader + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + if (accessor instanceof FieldVariableAccessor) { + LOG.info(name() + "\tbound as field attr reader " + Bootstrap.logMethod(method) + ":" + ((AttrReaderMethod)method).getVariableName()); + } else { + LOG.info(name() + "\tbound as attr reader " + Bootstrap.logMethod(method) + ":" + ((AttrReaderMethod)method).getVariableName()); + } + } + + return createAttrReaderHandle(self, self.getType(), accessor); + } else if (method instanceof AttrWriterMethod && arity == 1) { + AttrWriterMethod attrReader = (AttrWriterMethod)method; + String varName = attrReader.getVariableName(); + + // we getVariableAccessorForWrite here so it is eagerly created and we don't cache the DUMMY + VariableAccessor accessor = self.getType().getVariableAccessorForWrite(varName); + + // Ruby to attr reader + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + if (accessor instanceof FieldVariableAccessor) { + LOG.info(name() + "\tbound as field attr writer " + Bootstrap.logMethod(method) + ":" + ((AttrWriterMethod) method).getVariableName()); + } else { + LOG.info(name() + "\tbound as attr writer " + Bootstrap.logMethod(method) + ":" + ((AttrWriterMethod) method).getVariableName()); + } + } + + return createAttrWriterHandle(self, self.getType(), accessor); + } + + return null; + } + + private MethodHandle createAttrReaderHandle(IRubyObject self, RubyClass cls, VariableAccessor accessor) { + MethodHandle nativeTarget; + + MethodHandle filter = cls.getClassRuntime().getNullToNilHandle(); + + MethodHandle getValue; + + if (accessor instanceof FieldVariableAccessor) { + MethodHandle getter = ((FieldVariableAccessor)accessor).getGetter(); + getValue = Binder.from(type()) + .drop(0, argOffset - 1) + .filterReturn(filter) + .cast(methodType(Object.class, self.getClass())) + .invoke(getter); + } else { + getValue = Binder.from(type()) + .drop(0, argOffset - 1) + .filterReturn(filter) + .cast(methodType(Object.class, Object.class)) + .prepend(accessor) + .invokeVirtualQuiet(LOOKUP, "get"); + } + + // NOTE: Must not cache the fully-bound handle in the method, since it's specific to this class + + return getValue; + } + + private MethodHandle createAttrWriterHandle(IRubyObject self, RubyClass cls, VariableAccessor accessor) { + MethodHandle nativeTarget; + + MethodHandle filter = Binder + .from(IRubyObject.class, Object.class) + .drop(0) + .constant(cls.getRuntime().getNil()); + + MethodHandle setValue; + + if (accessor instanceof FieldVariableAccessor) { + MethodHandle setter = ((FieldVariableAccessor)accessor).getSetter(); + setValue = Binder.from(type()) + .drop(0, argOffset - 1) + .filterReturn(filter) + .cast(methodType(void.class, self.getClass(), Object.class)) + .invoke(setter); + } else { + setValue = Binder.from(type()) + .drop(0, argOffset - 1) + .filterReturn(filter) + .cast(methodType(void.class, Object.class, Object.class)) + .prepend(accessor) + .invokeVirtualQuiet(LOOKUP, "set"); + } + + return setValue; + } + + MethodHandle buildJittedHandle(CacheEntry entry, boolean blockGiven) { + MethodHandle mh = null; + SmartBinder binder; + CompiledIRMethod compiledIRMethod = null; + DynamicMethod method = entry.method; + RubyModule sourceModule = entry.sourceModule; + + if (method instanceof CompiledIRMethod) { + compiledIRMethod = (CompiledIRMethod)method; + } else if (method instanceof MixedModeIRMethod) { + DynamicMethod actualMethod = ((MixedModeIRMethod)method).getActualMethod(); + if (actualMethod instanceof CompiledIRMethod) { + compiledIRMethod = (CompiledIRMethod) actualMethod; + } else { + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(name() + "\tfailed direct binding due to unjitted method " + Bootstrap.logMethod(method)); + } + } + } + + if (compiledIRMethod != null) { + + // attempt IR direct binding + // TODO: this will have to expand when we start specializing arities + + binder = SmartBinder.from(signature) + .permute("context", "self", "arg.*", "block"); + + if (arity == -1) { + // already [], nothing to do + mh = (MethodHandle)compiledIRMethod.getHandle(); + } else if (arity == 0) { + MethodHandle specific; + if ((specific = compiledIRMethod.getHandleFor(arity)) != null) { + mh = specific; + } else { + mh = (MethodHandle)compiledIRMethod.getHandle(); + binder = binder.insert(2, "args", IRubyObject.NULL_ARRAY); + } + } else { + MethodHandle specific; + if ((specific = compiledIRMethod.getHandleFor(arity)) != null) { + mh = specific; + } else { + mh = (MethodHandle) compiledIRMethod.getHandle(); + binder = binder.collect("args", "arg.*", Helpers.constructObjectArrayHandle(arity)); + } + } + + if (!blockGiven) { + binder = binder.append("block", Block.class, Block.NULL_BLOCK); + } + + binder = binder + .insert(1, "scope", StaticScope.class, compiledIRMethod.getStaticScope()) + .append("class", RubyModule.class, sourceModule) + .append("frameName", String.class, name()); + + mh = binder.invoke(mh).handle(); + + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(name() + "\tbound directly to jitted method " + Bootstrap.logMethod(method)); + } + } + + return mh; + } + + MethodHandle buildNativeHandle(CacheEntry entry, boolean blockGiven) { + MethodHandle mh = null; + SmartBinder binder = null; + DynamicMethod method = entry.method; + + if (method instanceof NativeCallMethod && ((NativeCallMethod) method).getNativeCall() != null) { + NativeCallMethod nativeMethod = (NativeCallMethod)method; + DynamicMethod.NativeCall nativeCall = nativeMethod.getNativeCall(); + + DynamicMethod.NativeCall nc = nativeCall; + + if (nc.isJava()) { + return JavaBootstrap.createJavaHandle(this, method); + } else { + int nativeArgCount = getNativeArgCount(method, nativeCall); + + if (nativeArgCount >= 0) { // native methods only support arity 3 + if (nativeArgCount == arity) { + // nothing to do + binder = SmartBinder.from(lookup(), signature); + } else { + // arity mismatch...leave null and use DynamicMethod.call below + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(name() + "\tdid not match the primary arity for a native method " + Bootstrap.logMethod(method)); + } + } + } else { + // varargs + if (arity == -1) { + // ok, already passing [] + binder = SmartBinder.from(lookup(), signature); + } else if (arity == 0) { + // no args, insert dummy + binder = SmartBinder.from(lookup(), signature) + .insert(argOffset, "args", IRubyObject.NULL_ARRAY); + } else { + // 1 or more args, collect into [] + binder = SmartBinder.from(lookup(), signature) + .collect("args", "arg.*", Helpers.constructObjectArrayHandle(arity)); + } + } + + if (binder != null) { + + // clean up non-arguments, ordering, types + if (!nc.hasContext()) { + binder = binder.drop("context"); + } + + if (nc.hasBlock() && !blockGiven) { + binder = binder.append("block", Block.NULL_BLOCK); + } else if (!nc.hasBlock() && blockGiven) { + binder = binder.drop("block"); + } + + if (nc.isStatic()) { + mh = binder + .permute("context", "self", "arg.*", "block") // filter caller + .cast(nc.getNativeReturn(), nc.getNativeSignature()) + .invokeStaticQuiet(LOOKUP, nc.getNativeTarget(), nc.getNativeName()) + .handle(); + } else { + mh = binder + .permute("self", "context", "arg.*", "block") // filter caller, move self + .castArg("self", nc.getNativeTarget()) + .castVirtual(nc.getNativeReturn(), nc.getNativeTarget(), nc.getNativeSignature()) + .invokeVirtualQuiet(LOOKUP, nc.getNativeName()) + .handle(); + } + + if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) { + LOG.info(name() + "\tbound directly to JVM method " + Bootstrap.logMethod(method)); + } + + JRubyMethod anno = nativeCall.getMethod().getAnnotation(JRubyMethod.class); + if (anno != null && anno.frame()) { + mh = InvocationLinker.wrapWithFrameOnly(signature, entry.sourceModule, name(), mh); + } + } + } + } + + return mh; + } + + public static int getNativeArgCount(DynamicMethod method, DynamicMethod.NativeCall nativeCall) { + // if non-Java, must: + // * exactly match arities or both are [] boxed + // * 3 or fewer arguments + return getArgCount(nativeCall.getNativeSignature(), nativeCall.isStatic()); + } + + private static int getArgCount(Class[] args, boolean isStatic) { + int length = args.length; + boolean hasContext = false; + if (isStatic) { + if (args.length > 1 && args[0] == ThreadContext.class) { + length--; + hasContext = true; + } + + // remove self object + assert args.length >= 1; + length--; + + if (args.length > 1 && args[args.length - 1] == Block.class) { + length--; + } + + if (length == 1) { + if (hasContext && args[2] == IRubyObject[].class) { + length = -1; + } else if (args[1] == IRubyObject[].class) { + length = -1; + } + } + } else { + if (args.length > 0 && args[0] == ThreadContext.class) { + length--; + hasContext = true; + } + + if (args.length > 0 && args[args.length - 1] == Block.class) { + length--; + } + + if (length == 1) { + if (hasContext && args[1] == IRubyObject[].class) { + length = -1; + } else if (args[0] == IRubyObject[].class) { + length = -1; + } + } + } + return length; + } + public String name() { return methodName; } @@ -145,7 +625,7 @@ public InvokeSite(MethodType type, String name, CallType callType, boolean liter this.arity = arity; - this.fallback = prepareBinder(true).invokeVirtualQuiet(Bootstrap.LOOKUP, "invoke"); + this.fallback = prepareBinder(true).invokeVirtualQuiet(LOOKUP, "invoke"); } public static CallSite bootstrap(InvokeSite site, MethodHandles.Lookup lookup) { @@ -167,7 +647,7 @@ public IRubyObject invoke(ThreadContext context, IRubyObject caller, IRubyObject // only pass symbol below if we be calling a user-defined method_missing (default ones do it for us) passSymbol = !(entry.method instanceof RubyKernel.MethodMissingMethod || entry.method instanceof Helpers.MethodMissingWrapper); - mh = Bootstrap.buildGenericHandle(this, entry); + mh = buildGenericHandle(entry); } else { mh = getHandle(self, entry); } @@ -190,7 +670,7 @@ public IRubyObject invoke(ThreadContext context, IRubyObject self, IRubyObject[] // only pass symbol below if we be calling a user-defined method_missing (default ones do it for us) passSymbol = !(entry.method instanceof RubyKernel.MethodMissingMethod || entry.method instanceof Helpers.MethodMissingWrapper); - mh = Bootstrap.buildGenericHandle(this, entry); + mh = buildGenericHandle(entry); } else { mh = getHandle(self, entry); } @@ -523,26 +1003,26 @@ protected MethodHandle getHandle(IRubyObject self, CacheEntry entry) throws Thro MethodHandle mh = buildNewInstanceHandle(entry, self); if (mh == null) mh = buildNotEqualHandle(entry, self); - if (mh == null) mh = Bootstrap.buildNativeHandle(this, entry, blockGiven); - if (mh == null) mh = buildJavaFieldHandle(this, entry, self); - if (mh == null) mh = Bootstrap.buildIndyHandle(this, entry); - if (mh == null) mh = Bootstrap.buildJittedHandle(this, entry, blockGiven); - if (mh == null) mh = Bootstrap.buildAttrHandle(this, entry, self); + if (mh == null) mh = buildNativeHandle(entry, blockGiven); + if (mh == null) mh = buildJavaFieldHandle(entry, self); + if (mh == null) mh = buildIndyHandle(entry); + if (mh == null) mh = buildJittedHandle(entry, blockGiven); + if (mh == null) mh = buildAttrHandle(entry, self); if (mh == null) mh = buildAliasHandle(entry, self); if (mh == null) mh = buildStructHandle(entry); - if (mh == null) mh = Bootstrap.buildGenericHandle(this, entry); + if (mh == null) mh = buildGenericHandle(entry); assert mh != null : "we should have a method handle of some sort by now"; return mh; } - MethodHandle buildJavaFieldHandle(InvokeSite site, CacheEntry entry, IRubyObject self) throws Throwable { + MethodHandle buildJavaFieldHandle(CacheEntry entry, IRubyObject self) throws Throwable { DynamicMethod method = entry.method; if (method instanceof InstanceFieldGetter) { // only matching arity - if (site.arity != 0 || site.signature.lastArgType() == Block.class) return null; + if (arity != 0 || signature.lastArgType() == Block.class) return null; Field field = ((InstanceFieldGetter) method).getField(); @@ -565,7 +1045,7 @@ MethodHandle buildJavaFieldHandle(InvokeSite site, CacheEntry entry, IRubyObject .invokeStaticQuiet(lookup(), JavaUtil.class, "objectFromJavaProxy"); fieldHandle = Binder - .from(site.type()) + .from(type()) .permute(2) .filter(0, receiverConverter) .filterReturn(filter) @@ -577,7 +1057,7 @@ MethodHandle buildJavaFieldHandle(InvokeSite site, CacheEntry entry, IRubyObject return fieldHandle; } else if (method instanceof InstanceFieldSetter) { // only matching arity - if (site.arity != 1 || site.signature.lastArgType() == Block.class) return null; + if (arity != 1 || signature.lastArgType() == Block.class) return null; Field field = ((InstanceFieldSetter) method).getField(); @@ -598,7 +1078,7 @@ MethodHandle buildJavaFieldHandle(InvokeSite site, CacheEntry entry, IRubyObject .invokeStaticQuiet(lookup(), JavaUtil.class, "objectFromJavaProxy"); fieldHandle = Binder - .from(site.type()) + .from(type()) .permute(2, 3) .filter(0, receiverConverter) .filterReturn(constant(IRubyObject.class, self.getRuntime().getNil())) @@ -842,7 +1322,7 @@ protected SmartHandle testTarget(IRubyObject self, RubyModule testClass) { .from(signature.changeReturn(boolean.class)) .permute("self") .insert(0, "selfClass", RubyClass.class, testClass) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "testType"); + .invokeStaticQuiet(LOOKUP, InvokeSite.class, "testType"); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IsNilSite.java b/core/src/main/java/org/jruby/ir/targets/indy/IsNilSite.java new file mode 100644 index 00000000000..77c7bc96c89 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/IsNilSite.java @@ -0,0 +1,60 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.RubyNil; +import org.jruby.runtime.builtin.IRubyObject; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; + +import static java.lang.invoke.MethodHandles.insertArguments; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class IsNilSite extends MutableCallSite { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + public static final MethodType TYPE = methodType(boolean.class, IRubyObject.class); + public static final Handle IS_NIL_BOOTSTRAP_HANDLE = new Handle( + Opcodes.H_INVOKESTATIC, + p(IsNilSite.class), + "isNil", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class), + false); + + private static final MethodHandle INIT_HANDLE = + Binder.from(TYPE.insertParameterTypes(0, IsNilSite.class)).invokeVirtualQuiet(LOOKUP, "init"); + + private static final MethodHandle IS_NIL_HANDLE = + Binder + .from(boolean.class, IRubyObject.class, RubyNil.class) + .invokeStaticQuiet(LOOKUP, IsNilSite.class, "isNil"); + + public IsNilSite() { + super(TYPE); + + setTarget(INIT_HANDLE.bindTo(this)); + } + + public static CallSite isNil(MethodHandles.Lookup lookup, String name, MethodType type) { + return new IsNilSite(); + } + + public boolean init(IRubyObject obj) { + IRubyObject nil = obj.getRuntime().getNil(); + + setTarget(insertArguments(IS_NIL_HANDLE, 1, nil)); + + return nil == obj; + } + + public static boolean isNil(IRubyObject obj, RubyNil nil) { + return nil == obj; + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IsTrueSite.java b/core/src/main/java/org/jruby/ir/targets/indy/IsTrueSite.java new file mode 100644 index 00000000000..451313f261a --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/IsTrueSite.java @@ -0,0 +1,64 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.Ruby; +import org.jruby.RubyBoolean; +import org.jruby.RubyNil; +import org.jruby.runtime.builtin.IRubyObject; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; + +import static java.lang.invoke.MethodHandles.insertArguments; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class IsTrueSite extends MutableCallSite { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + public static final MethodType TYPE = methodType(boolean.class, IRubyObject.class); + + public static final MethodHandle INIT_HANDLE = Binder.from(TYPE.insertParameterTypes(0, IsTrueSite.class)).invokeVirtualQuiet(LOOKUP, "init"); + public static final Handle IS_TRUE_BOOTSTRAP_HANDLE = new Handle( + Opcodes.H_INVOKESTATIC, + p(IsTrueSite.class), + "isTrue", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class), + false); + + private static final MethodHandle IS_TRUE_HANDLE = + Binder + .from(boolean.class, IRubyObject.class, RubyNil.class, RubyBoolean.False.class) + .invokeStaticQuiet(LOOKUP, IsTrueSite.class, "isTruthy"); + + public IsTrueSite() { + super(TYPE); + + setTarget(INIT_HANDLE.bindTo(this)); + } + + public static CallSite isTrue(MethodHandles.Lookup lookup, String name, MethodType type) { + return new IsTrueSite(); + } + + public boolean init(IRubyObject obj) { + Ruby runtime = obj.getRuntime(); + + IRubyObject nil = runtime.getNil(); + IRubyObject fals = runtime.getFalse(); + + setTarget(insertArguments(IS_TRUE_HANDLE, 1, nil, fals)); + + return nil != obj && fals != obj; + } + + public static boolean isTruthy(IRubyObject obj, RubyNil nil, RubyBoolean.False fals) { + return nil != obj && fals != obj; + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/simple/NormalBranchCompiler.java b/core/src/main/java/org/jruby/ir/targets/simple/NormalBranchCompiler.java index 499d8c71521..35e7d9edb2f 100644 --- a/core/src/main/java/org/jruby/ir/targets/simple/NormalBranchCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/simple/NormalBranchCompiler.java @@ -4,6 +4,7 @@ import org.jruby.ir.targets.BranchCompiler; import org.jruby.ir.targets.IRBytecodeAdapter; import org.jruby.ir.targets.indy.Bootstrap; +import org.jruby.ir.targets.indy.CheckArityBootstrap; import org.jruby.parser.StaticScope; import org.jruby.runtime.Block; import org.jruby.runtime.ThreadContext; @@ -55,10 +56,11 @@ public void checkArgsArity(Runnable args, int required, int opt, boolean rest) { public void checkArity(int required, int opt, boolean rest, int restKey) { compiler.adapter.ldc(required); + compiler.adapter.ldc(opt); compiler.adapter.ldc(rest); compiler.adapter.ldc(restKey); - compiler.adapter.invokestatic(p(Bootstrap.class), "checkArity", sig(void.class, params(ThreadContext.class, StaticScope.class, Object[].class, Object.class, Block.class, int.class, int.class, boolean.class, int.class))); + compiler.adapter.invokestatic(p(CheckArityBootstrap.class), "checkArity", sig(void.class, params(ThreadContext.class, StaticScope.class, Object[].class, Object.class, Block.class, int.class, int.class, boolean.class, int.class))); } public void checkAritySpecificArgs(int required, int opt, boolean rest, int restKey) { @@ -66,6 +68,6 @@ public void checkAritySpecificArgs(int required, int opt, boolean rest, int rest compiler.adapter.ldc(opt); compiler.adapter.ldc(rest); compiler.adapter.ldc(restKey); - compiler.adapter.invokestatic(p(Bootstrap.class), "checkAritySpecificArgs", sig(void.class, params(ThreadContext.class, StaticScope.class, Object[].class, Block.class, int.class, int.class, boolean.class, int.class))); + compiler.adapter.invokestatic(p(CheckArityBootstrap.class), "checkAritySpecificArgs", sig(void.class, params(ThreadContext.class, StaticScope.class, Object[].class, Block.class, int.class, int.class, boolean.class, int.class))); } } diff --git a/core/src/main/java/org/jruby/runtime/invokedynamic/GlobalSite.java b/core/src/main/java/org/jruby/runtime/invokedynamic/GlobalSite.java index 3a8f79e6a34..b44bbf16819 100644 --- a/core/src/main/java/org/jruby/runtime/invokedynamic/GlobalSite.java +++ b/core/src/main/java/org/jruby/runtime/invokedynamic/GlobalSite.java @@ -1,16 +1,40 @@ package org.jruby.runtime.invokedynamic; +import java.lang.invoke.CallSite; import java.lang.invoke.MethodHandle; import org.jruby.Ruby; -import org.jruby.RubyClass; -import org.jruby.common.IRubyWarnings; +import org.jruby.RubyGlobal; +import org.jruby.internal.runtime.GlobalVariable; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.opto.Invalidator; +import org.jruby.util.JavaNameMangler; +import org.jruby.util.cli.Options; +import org.jruby.util.log.Logger; +import org.jruby.util.log.LoggerFactory; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; +import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.lang.invoke.MutableCallSite; +import java.lang.invoke.SwitchPoint; + +import static java.lang.invoke.MethodHandles.constant; +import static java.lang.invoke.MethodHandles.dropArguments; +import static java.lang.invoke.MethodHandles.lookup; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; public class GlobalSite extends MutableCallSite { + public static final Handle GLOBAL_BOOTSTRAP_H = new Handle( + Opcodes.H_INVOKESTATIC, + p(GlobalSite.class), + "globalBootstrap", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, int.class), + false); + private static final Logger LOG = LoggerFactory.getLogger(GlobalSite.class); private final String name; private volatile int failures; private final String file; @@ -22,16 +46,16 @@ public GlobalSite(MethodType type, String name, String file, int line) { this.file = file; this.line = line; } - + public void setTarget(MethodHandle target) { super.setTarget(target); incrementFailures(); } - + public int failures() { return failures; } - + public void incrementFailures() { failures += 1; } @@ -43,4 +67,79 @@ public String name() { public String file() { return file; } public int line() { return line; } + + public static CallSite globalBootstrap(MethodHandles.Lookup lookup, String name, MethodType type, String file, int line) throws Throwable { + String[] names = name.split(":"); + String operation = names[0]; + String varName = JavaNameMangler.demangleMethodName(names[1]); + GlobalSite site = new GlobalSite(type, varName, file, line); + MethodHandle handle; + + if (operation.equals("get")) { + handle = lookup.findVirtual(GlobalSite.class, "getGlobalFallback", methodType(IRubyObject.class, ThreadContext.class)); + } else { + handle = lookup.findVirtual(GlobalSite.class, "setGlobalFallback", methodType(void.class, IRubyObject.class, ThreadContext.class)); + } + + handle = handle.bindTo(site); + site.setTarget(handle); + + return site; + } + + public IRubyObject getGlobalFallback(ThreadContext context) throws Throwable { + Ruby runtime = context.runtime; + GlobalVariable variable = runtime.getGlobalVariables().getVariable(name()); + + if (failures() > Options.INVOKEDYNAMIC_GLOBAL_MAXFAIL.load() || + variable.getScope() != GlobalVariable.Scope.GLOBAL || + RubyGlobal.UNCACHED_GLOBALS.contains(name())) { + + // use uncached logic forever + if (Options.INVOKEDYNAMIC_LOG_GLOBALS.load()) LOG.info("global " + name() + " (" + file() + ":" + line() + ") uncacheable or rebound > " + Options.INVOKEDYNAMIC_GLOBAL_MAXFAIL.load() + " times, reverting to simple lookup"); + + MethodHandle uncached = lookup().findStatic(GlobalSite.class, "getGlobalUncached", methodType(IRubyObject.class, GlobalVariable.class)); + uncached = uncached.bindTo(variable); + uncached = dropArguments(uncached, 0, ThreadContext.class); + setTarget(uncached); + return (IRubyObject)uncached.invokeWithArguments(context); + } + + Invalidator invalidator = variable.getInvalidator(); + IRubyObject value = variable.getAccessor().getValue(); + + MethodHandle target = constant(IRubyObject.class, value); + target = dropArguments(target, 0, ThreadContext.class); + MethodHandle fallback = lookup().findVirtual(GlobalSite.class, "getGlobalFallback", methodType(IRubyObject.class, ThreadContext.class)); + fallback = fallback.bindTo(this); + + target = ((SwitchPoint)invalidator.getData()).guardWithTest(target, fallback); + + setTarget(target); + +// if (Options.INVOKEDYNAMIC_LOG_GLOBALS.load()) LOG.info("global " + name() + " (" + file() + ":" + line() + ") cached"); + + return value; + } + + public static IRubyObject getGlobalUncached(GlobalVariable variable) throws Throwable { + return variable.getAccessor().getValue(); + } + + public void setGlobalFallback(IRubyObject value, ThreadContext context) throws Throwable { + Ruby runtime = context.runtime; + GlobalVariable variable = runtime.getGlobalVariables().getVariable(name()); + MethodHandle uncached = lookup().findStatic(GlobalSite.class, "setGlobalUncached", methodType(void.class, GlobalVariable.class, IRubyObject.class)); + uncached = uncached.bindTo(variable); + uncached = dropArguments(uncached, 1, ThreadContext.class); + setTarget(uncached); + uncached.invokeWithArguments(value, context); + } + + public static void setGlobalUncached(GlobalVariable variable, IRubyObject value) throws Throwable { + // FIXME: duplicated logic from GlobalVariables.set + variable.getAccessor().setValue(value); + variable.trace(value); + variable.invalidate(); + } } From ade997d142ffeb898616096dd219560e291e93c1 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 15 Sep 2023 11:29:18 +0200 Subject: [PATCH 06/56] Remove deprecated methods --- core/src/main/java/org/jruby/RubyThread.java | 99 +------------------- 1 file changed, 2 insertions(+), 97 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyThread.java b/core/src/main/java/org/jruby/RubyThread.java index 69eb9201004..35f425b16a5 100644 --- a/core/src/main/java/org/jruby/RubyThread.java +++ b/core/src/main/java/org/jruby/RubyThread.java @@ -212,6 +212,8 @@ public enum Status { /** Short circuit to avoid-re-scanning for interrupts */ private volatile boolean pendingInterruptQueueChecked = false; + private volatile BlockingTask currentBlockingTask; + private volatile Selector currentSelector; private volatile RubyThread fiberCurrentThread; @@ -2557,101 +2559,4 @@ public static IRubyObject uninterruptible(ThreadContext context, Sta state, f); } - - private static final String MUTEX_FOR_THREAD_EXCLUSIVE = "MUTEX_FOR_THREAD_EXCLUSIVE"; - - @Deprecated // Thread.exclusive(&block) - @JRubyMethod(meta = true) - public static IRubyObject exclusive(ThreadContext context, IRubyObject recv, Block block) { - recv.callMethod(context, "warn", context.runtime.newString("Thread.exclusive is deprecated, use Thread::Mutex")); - return getMutexForThreadExclusive(context, (RubyClass) recv).synchronize(context, block); - } - - private static Mutex getMutexForThreadExclusive(ThreadContext context, RubyClass recv) { - Mutex mutex = (Mutex) recv.getConstantNoConstMissing(MUTEX_FOR_THREAD_EXCLUSIVE, false, false); - if (mutex != null) return mutex; - synchronized (recv) { - mutex = (Mutex) recv.getConstantNoConstMissing(MUTEX_FOR_THREAD_EXCLUSIVE, false, false); - if (mutex == null) { - mutex = Mutex.newInstance(context, context.runtime.getMutex(), NULL_ARRAY, Block.NULL_BLOCK); - recv.setConstant(MUTEX_FOR_THREAD_EXCLUSIVE, mutex, true); - } - return mutex; - } - } - - /** - * This is intended to be used to raise exceptions in Ruby threads from non- - * Ruby threads like Timeout's thread. - * - * @param args Same args as for Thread#raise - */ - @Deprecated - public void internalRaise(IRubyObject[] args) { - ThreadContext context = getRuntime().getCurrentContext(); - genericRaise(context, context.getThread(), args); - } - - @Deprecated - public void receiveMail(ThreadService.Event event) { - } - - @Deprecated - public void checkMail(ThreadContext context) { - } - - @Deprecated - private volatile BlockingTask currentBlockingTask; - - @Deprecated - public boolean selectForAccept(RubyIO io) { - return select(io, SelectionKey.OP_ACCEPT); - } - - @Deprecated - public IRubyObject backtrace20(ThreadContext context, IRubyObject[] args) { - return backtrace(context); - } - - @Deprecated - public IRubyObject backtrace(ThreadContext context, IRubyObject[] args) { - switch (args.length) { - case 0: - return backtrace(context); - case 1: - return backtrace(context, args[0]); - case 2: - return backtrace(context, args[0], args[1]); - default: - Arity.checkArgumentCount(context.runtime, args, 0, 2); - return null; // not reached - } - } - - @Deprecated - public IRubyObject backtrace_locations(ThreadContext context, IRubyObject[] args) { - switch (args.length) { - case 0: - return backtrace_locations(context); - case 1: - return backtrace_locations(context, args[0]); - case 2: - return backtrace_locations(context, args[0], args[1]); - default: - Arity.checkArgumentCount(context.runtime, args, 0, 2); - return null; // not reached - } - } - - @Deprecated - public static IRubyObject pass(IRubyObject recv) { - Ruby runtime = recv.getRuntime(); - - return pass(runtime.getCurrentContext(), recv); - } - - @Deprecated - public IRubyObject safe_level() { - throw getRuntime().newNotImplementedError("Thread-specific SAFE levels are not supported"); - } } From e05ed37316a39d91887cfaeeb0a793ab06d53d98 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 15 Sep 2023 15:59:24 +0200 Subject: [PATCH 07/56] First pass adding scheduler support * Only IO endpoints calling back into scheduler so far * IO::Buffer totally stubbed out * Organization of utility methods likely to change --- .../main/java/org/jruby/FiberScheduler.java | 157 ++++++ core/src/main/java/org/jruby/Ruby.java | 27 + core/src/main/java/org/jruby/RubyIO.java | 6 + .../src/main/java/org/jruby/RubyIOBuffer.java | 484 ++++++++++++++++++ core/src/main/java/org/jruby/RubyThread.java | 42 +- .../java/org/jruby/ext/fiber/ThreadFiber.java | 34 ++ .../main/java/org/jruby/util/io/OpenFile.java | 36 +- 7 files changed, 781 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/jruby/FiberScheduler.java create mode 100644 core/src/main/java/org/jruby/RubyIOBuffer.java diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java new file mode 100644 index 00000000000..3d2ad2f63aa --- /dev/null +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -0,0 +1,157 @@ +package org.jruby; + +import org.jruby.runtime.Helpers; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.io.OpenFile; + +public class FiberScheduler { + // MRI: rb_fiber_scheduler_kernel_sleep + public static IRubyObject kernelSleep(ThreadContext context, IRubyObject scheduler, IRubyObject timeout) { + return scheduler.callMethod(context, "kernel_sleep", timeout); + } + + // MRI: rb_fiber_scheduler_kernel_sleepv + public static IRubyObject kernelSleep(ThreadContext context, IRubyObject scheduler, IRubyObject[] args) { + return scheduler.callMethod(context, "kernel_sleep", args); + } + + // MRI: rb_fiber_scheduler_process_wait + public static IRubyObject processWait(ThreadContext context, IRubyObject scheduler, long pid, int flags) { + return Helpers.invokeChecked(context, scheduler, "process_wait", context.runtime.newFixnum(pid), context.runtime.newFixnum(flags)); + } + + // MRI: rb_fiber_scheduler_block + public static IRubyObject block(ThreadContext context, IRubyObject scheduler, IRubyObject blocker, IRubyObject timeout) { + return Helpers.invoke(context, scheduler, "block", blocker, timeout); + } + + // MRI: rb_fiber_scheduler_unblock + public static IRubyObject unblock(ThreadContext context, IRubyObject scheduler, IRubyObject blocker, IRubyObject fiber) { + return Helpers.invoke(context, scheduler, "unblock", blocker, fiber); + } + + // MRI: rb_fiber_scheduler_io_wait + public static IRubyObject ioWait(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject events, IRubyObject timeout) { + return Helpers.invoke(context, scheduler, "io_wait", io, events, timeout); + } + + // MRI: rb_fiber_scheduler_io_wait_readable + public static IRubyObject ioWaitReadable(ThreadContext context, IRubyObject scheduler, IRubyObject io) { + return ioWait(context, scheduler, io, context.runtime.newFixnum(OpenFile.READABLE), context.nil); + } + + // MRI: rb_fiber_scheduler_io_wait_writable + public static IRubyObject ioWaitWritable(ThreadContext context, IRubyObject scheduler, IRubyObject io) { + return ioWait(context, scheduler, io, context.runtime.newFixnum(OpenFile.WRITABLE), context.nil); + } + + // MRI: rb_fiber_scheduler_io_select + public static IRubyObject ioSelect(ThreadContext context, IRubyObject scheduler, IRubyObject readables, IRubyObject writables, IRubyObject exceptables, IRubyObject timeout) { + return ioSelectv(context, scheduler, readables, writables, exceptables, timeout); + } + + // MRI: rb_fiber_scheduler_io_selectv + public static IRubyObject ioSelectv(ThreadContext context, IRubyObject scheduler, IRubyObject... args) { + return Helpers.invokeChecked(context, scheduler, "io_select", args); + } + + // MRI: rb_fiber_scheduler_io_read + public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length) { + return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, context.runtime.newFixnum(length)); + } + + // MRI: rb_fiber_scheduler_io_pread + public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { + return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); + } + + // MRI: rb_fiber_scheduler_io_write + public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length) { + return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, context.runtime.newFixnum(length)); + } + + // MRI: rb_fiber_scheduler_io_pwrite + public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { + return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); + } + + // MRI: rb_fiber_scheduler_io_read_memory + public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, byte[] base, int size, int length) { + RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED); + + IRubyObject result = ioRead(context, scheduler, io, buffer, length); + + buffer.unlock(context); + buffer.free(context); + + return result; + } + + // MRI: rb_fiber_scheduler_io_write_memory + public static IRubyObject ioWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, byte[] base, int size, int length) { + RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); + + IRubyObject result = ioWrite(context, scheduler, io, buffer, length); + + buffer.unlock(context); + buffer.free(context); + + return result; + } + + // MRI: rb_fiber_scheduler_io_close + public static IRubyObject ioClose(ThreadContext context, IRubyObject scheduler, IRubyObject io) { + return Helpers.invokeChecked(context, scheduler, "io_close", io); + } + + // MRI: rb_fiber_scheduler_address_resolve + public static IRubyObject addressResolve(ThreadContext context, IRubyObject scheduler, IRubyObject hostname) { + return Helpers.invokeChecked(context, scheduler, "address_resolve", hostname); + } + + // MRI: verify_scheduler + static void verifyInterface(IRubyObject scheduler) { + if (!scheduler.respondsTo("block")) { + throw scheduler.getRuntime().newArgumentError("Scheduler must implement #block"); + } + + if (!scheduler.respondsTo("unblock")) { + throw scheduler.getRuntime().newArgumentError("Scheduler must implement #unblock"); + } + + if (!scheduler.respondsTo("kernel_sleep")) { + throw scheduler.getRuntime().newArgumentError("Scheduler must implement #kernel_sleep"); + } + + if (!scheduler.respondsTo("io_wait")) { + throw scheduler.getRuntime().newArgumentError("Scheduler must implement #io_wait"); + } + } + + // MRI: rb_fiber_scheduler_close + public static IRubyObject close(ThreadContext context, IRubyObject scheduler) { +// VM_ASSERT(ruby_thread_has_gvl_p()); + + IRubyObject result; + + result = Helpers.invokeChecked(context, scheduler, "scheduler_close"); + if (result != RubyBasicObject.UNDEF) return result; + + result = Helpers.invokeChecked(context, scheduler, "close"); + if (result != RubyBasicObject.UNDEF) return result; + + return context.nil; + } + + // MRI: rb_fiber_scheduler_io_result_apply + public static int resultApply(ThreadContext context, IRubyObject result) { + int resultInt; + if (result instanceof RubyFixnum && (resultInt = RubyNumeric.num2int(result)) < 0) { + context.runtime.getPosix().errno(-resultInt); + return -1; + } else { + return RubyNumeric.num2int(result); + } + } +} diff --git a/core/src/main/java/org/jruby/Ruby.java b/core/src/main/java/org/jruby/Ruby.java index 0a65b89ddde..eaa8ebebe99 100644 --- a/core/src/main/java/org/jruby/Ruby.java +++ b/core/src/main/java/org/jruby/Ruby.java @@ -443,6 +443,7 @@ private Ruby(RubyInstanceConfig config) { randomClass = null; } ioClass = RubyIO.createIOClass(this); + ioBufferClass = RubyIOBuffer.createIOBufferClass(this); structClass = profile.allowClass("Struct") ? RubyStruct.createStructClass(this) : null; bindingClass = profile.allowClass("Binding") ? RubyBinding.createBindingClass(this) : null; @@ -1714,6 +1715,14 @@ private void initExceptions() { ifAllowed("KeyError", (ruby) -> keyError = RubyKeyError.define(ruby, indexError)); ifAllowed("DomainError", (ruby) -> mathDomainError = RubyDomainError.define(ruby, argumentError, mathModule)); + RubyClass runtimeError = this.runtimeError; + ObjectAllocator runtimeErrorAllocator = runtimeError.getAllocator(); + bufferLockedError = ioBufferClass.defineClassUnder("LockedError", runtimeError, runtimeErrorAllocator); + ioBufferClass.defineClassUnder("AllocationError", runtimeError, runtimeErrorAllocator); + ioBufferClass.defineClassUnder("AccessError", runtimeError, runtimeErrorAllocator); + ioBufferClass.defineClassUnder("InvalidatedError", runtimeError, runtimeErrorAllocator); + ioBufferClass.defineClassUnder("MaskError", runtimeError, runtimeErrorAllocator); + initErrno(); initNativeException(); @@ -2174,6 +2183,10 @@ public RubyClass getIO() { return ioClass; } + public RubyClass getIOBuffer() { + return ioBufferClass; + } + public RubyClass getThread() { return threadClass; } @@ -2485,6 +2498,10 @@ public RubyClass getInvalidByteSequenceError() { return invalidByteSequenceError; } + public RubyClass getBufferLockedError() { + return bufferLockedError; + } + @Deprecated RubyRandom.RandomType defaultRand; @@ -4246,6 +4263,10 @@ public RaiseException newInvalidByteSequenceError(String message) { return newRaiseException(getInvalidByteSequenceError(), message); } + public RaiseException newBufferLockedError(String message) { + return newRaiseException(getBufferLockedError(), message); + } + /** * Construct a new RaiseException wrapping a new Ruby exception object appropriate to the given exception class. * @@ -5361,6 +5382,7 @@ public RubyClass getData() { private final RubyClass fileClass; private final RubyClass fileStatClass; private final RubyClass ioClass; + private final RubyClass ioBufferClass; private final RubyClass threadClass; private final RubyClass threadGroupClass; private final RubyClass continuationClass; @@ -5422,6 +5444,11 @@ public RubyClass getData() { private RubyClass keyError; private RubyClass locationClass; private RubyClass interruptedRegexpError; + private RubyClass bufferLockedError; + private RubyClass bufferAllocationError; + private RubyClass bufferAccessError; + private RubyClass bufferInvalidatedError; + private RubyClass bufferMaskError; /** * All the core modules we keep direct references to, for quick access and diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java index 33361dcc8a4..55ca0416a9b 100644 --- a/core/src/main/java/org/jruby/RubyIO.java +++ b/core/src/main/java/org/jruby/RubyIO.java @@ -3794,6 +3794,12 @@ public static RubyIO convertToIO(ThreadContext context, IRubyObject obj) { @JRubyMethod(name = "select", required = 1, optional = 3, checkArity = false, meta = true) public static IRubyObject select(ThreadContext context, IRubyObject recv, IRubyObject[] argv) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioSelectv(context, scheduler, argv); + if (result != UNDEF) return result; + } + int argc = Arity.checkArgumentCount(context, argv, 1, 4); IRubyObject read, write, except, _timeout; diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java new file mode 100644 index 00000000000..47ca2969f35 --- /dev/null +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -0,0 +1,484 @@ +package org.jruby; + +import jnr.ffi.Platform; +import org.jruby.anno.JRubyConstant; +import org.jruby.anno.JRubyMethod; +import org.jruby.runtime.Arity; +import org.jruby.runtime.Block; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; + +public class RubyIOBuffer extends RubyObject { + public static RubyClass createIOBufferClass(Ruby runtime) { + RubyClass IOBuffer = runtime.getIO().defineClassUnder("Buffer", runtime.getObject(), RubyIOBuffer::new); + + IOBuffer.includeModule(runtime.getComparable()); + + IOBuffer.defineAnnotatedMethods(RubyIOBuffer.class); + IOBuffer.defineAnnotatedConstants(RubyIOBuffer.class); + + return IOBuffer; + } + + @JRubyConstant + public static final int PAGE_SIZE = 8196; + @JRubyConstant + public static final int DEFAULT_SIZE = 8196; + @JRubyConstant + public static final int EXTERNAL = 1; + @JRubyConstant + public static final int INTERNAL = 2; + @JRubyConstant + public static final int MAPPED = 4; + @JRubyConstant + public static final int SHARED = 8; + @JRubyConstant + public static final int LOCKED = 32; + @JRubyConstant + public static final int PRIVATE = 64; + @JRubyConstant + public static final int READONLY = 128; + @JRubyConstant + public static final int LITTLE_ENDIAN = 4; + @JRubyConstant + public static final int BIG_ENDIAN = 8; + @JRubyConstant + public static final int HOST_ENDIAN = Platform.getNativePlatform().isBigEndian() ? BIG_ENDIAN : LITTLE_ENDIAN; + @JRubyConstant + public static final int NETWORK_ENDIAN = BIG_ENDIAN; + + public static RubyIOBuffer newBuffer(Ruby runtime, byte[] base, int size, int flags) { + return new RubyIOBuffer(runtime, runtime.getIO(), base, size, flags); + } + + public RubyIOBuffer(Ruby runtime, RubyClass metaClass) { + super(runtime, metaClass); + } + + public RubyIOBuffer(Ruby runtime, RubyClass metaClass, byte[] base, int size, int flags) { + super(runtime, metaClass); + + this.base = base; + this.size = size; + this.flags = flags; + } + + @JRubyMethod(name = "for") + public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyObject string) { + return context.nil; + } + + @JRubyMethod(name = "initialize") + public IRubyObject initialize(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "initialize") + public IRubyObject initialize(ThreadContext context, IRubyObject size) { + return context.nil; + } + + @JRubyMethod(name = "initialize") + public IRubyObject initialize(ThreadContext context, IRubyObject size, IRubyObject flags) { + return context.nil; + } + + @JRubyMethod(name = "initialize_copy") + public IRubyObject initialize_copy(ThreadContext context, IRubyObject other) { + return context.nil; + } + + @JRubyMethod(name = "inspect") + public IRubyObject inspect(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "hexdump") + public IRubyObject hexdump(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "to_s") + public IRubyObject to_s(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "size") + public IRubyObject size(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "valid?") + public IRubyObject valid_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "transfer") + public IRubyObject transfer(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "null?") + public IRubyObject null_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "empty?") + public IRubyObject empty_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "external?") + public IRubyObject external_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "internal?") + public IRubyObject internal_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "mapped?") + public IRubyObject mapped_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "shared?") + public IRubyObject shared_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "locked?") + public IRubyObject locked_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "readonly?") + public IRubyObject readonly_p(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "locked") + public IRubyObject locked(ThreadContext context, Block block) { + return context.nil; + } + + public IRubyObject lock(ThreadContext context) { + if ((flags & LOCKED) == LOCKED) { + throw context.runtime.newBufferLockedError("Buffer already locked!"); + } + + flags |= LOCKED; + + return this; + } + + public IRubyObject unlock(ThreadContext context) { + if ((flags & LOCKED) == 0) { + throw context.runtime.newBufferLockedError("Buffer not locked!"); + } + + flags &= ~LOCKED; + + return this; + } + + @JRubyMethod(name = "slice") + public IRubyObject slice(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "slice") + public IRubyObject slice(ThreadContext context, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "slice") + public IRubyObject slice(ThreadContext context, IRubyObject offset, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "<=>") + public IRubyObject op_cmp(ThreadContext context, IRubyObject other) { + return context.nil; + } + + @JRubyMethod(name = "resize") + public IRubyObject resize(ThreadContext context, IRubyObject size) { + return context.nil; + } + + @JRubyMethod(name = "clear") + public IRubyObject clear(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "clear") + public IRubyObject clear(ThreadContext context, IRubyObject value) { + return context.nil; + } + + @JRubyMethod(name = "clear") + public IRubyObject clear(ThreadContext context, IRubyObject value, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "clear") + public IRubyObject clear(ThreadContext context, IRubyObject value, IRubyObject offset, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "free") + public IRubyObject free(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "size_of") + public static IRubyObject size_of(ThreadContext context, IRubyObject self, IRubyObject dataType) { + return context.nil; + } + + @JRubyMethod(name = "get_value") + public IRubyObject get_value(ThreadContext context, IRubyObject type, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "get_values") + public IRubyObject get_values(ThreadContext context, IRubyObject data_types, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "each") + public IRubyObject each(ThreadContext context, IRubyObject dataType, Block block) { + return context.nil; + } + + @JRubyMethod(name = "each") + public IRubyObject each(ThreadContext context, IRubyObject dataType, IRubyObject offset, Block block) { + return context.nil; + } + + @JRubyMethod(name = "each") + public IRubyObject each(ThreadContext context, IRubyObject dataType, IRubyObject offset, IRubyObject count, Block block) { + return context.nil; + } + + @JRubyMethod(name = "values") + public IRubyObject values(ThreadContext context, IRubyObject dataType) { + return context.nil; + } + + @JRubyMethod(name = "values") + public IRubyObject values(ThreadContext context, IRubyObject dataType, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "values") + public IRubyObject values(ThreadContext context, IRubyObject dataType, IRubyObject offset, IRubyObject count) { + return context.nil; + } + + @JRubyMethod(name = "each_byte") + public IRubyObject each_byte(ThreadContext context, Block block) { + return context.nil; + } + + @JRubyMethod(name = "each_byte") + public IRubyObject each_byte(ThreadContext context, IRubyObject offset, Block block) { + return context.nil; + } + + @JRubyMethod(name = "each_byte") + public IRubyObject each_byte(ThreadContext context, IRubyObject offset, IRubyObject count, Block block) { + return context.nil; + } + + @JRubyMethod(name = "set_value") + public IRubyObject set_value(ThreadContext context, IRubyObject dataType, IRubyObject offset, IRubyObject value) { + return context.nil; + } + + @JRubyMethod(name = "set_values") + public IRubyObject set_values(ThreadContext context, IRubyObject dataTypes, IRubyObject offset, IRubyObject values) { + return context.nil; + } + + @JRubyMethod(name = "copy") + public IRubyObject copy(ThreadContext context, IRubyObject source) { + return context.nil; + } + + @JRubyMethod(name = "copy") + public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "copy") + public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject offset, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "copy", required = 1, optional = 3) + public IRubyObject copy(ThreadContext context, IRubyObject[] args) { + Arity.checkArgumentCount(context, args, 1, 3); + + switch (args.length) { + case 1: + return copy(context, args[0]); + case 2: + return copy(context, args[0], args[1]); + case 3: + return copy(context, args[0], args[1], args[2]); + case 4: + return copy(context, args[0], args[1], args[2], args[3]); + } + + return context.nil; + } + + public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject offset, IRubyObject length, IRubyObject sourceOffset) { + return context.nil; + } + + @JRubyMethod(name = "get_string") + public IRubyObject get_string(ThreadContext context, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "get_string") + public IRubyObject get_string(ThreadContext context, IRubyObject offset, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "get_string") + public IRubyObject get_string(ThreadContext context, IRubyObject offset, IRubyObject length, IRubyObject encoding) { + return context.nil; + } + + @JRubyMethod(name = "set_string") + public IRubyObject set_string(ThreadContext context, IRubyObject string) { + return context.nil; + } + + @JRubyMethod(name = "set_string") + public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "set_string") + public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyObject offset, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "set_string", required = 1, optional = 3) + public IRubyObject set_string(ThreadContext context, IRubyObject[] args) { + Arity.checkArgumentCount(context, args, 1, 3); + + switch (args.length) { + case 1: + return set_string(context, args[0]); + case 2: + return set_string(context, args[0], args[1]); + case 3: + return set_string(context, args[0], args[1], args[2]); + case 4: + return set_string(context, args[0], args[1], args[2], args[3]); + } + + return context.nil; + } + + public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyObject offset, IRubyObject length, IRubyObject encoding) { + return context.nil; + } + + @JRubyMethod(name = "&") + public IRubyObject op_and(ThreadContext context, IRubyObject mask) { + return context.nil; + } + + @JRubyMethod(name = "|") + public IRubyObject op_or(ThreadContext context, IRubyObject mask) { + return context.nil; + } + + @JRubyMethod(name = "^") + public IRubyObject op_xor(ThreadContext context, IRubyObject mask) { + return context.nil; + } + + @JRubyMethod(name = "~") + public IRubyObject op_not(ThreadContext context) { + return context.nil; + } + + @JRubyMethod(name = "read") + public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "read") + public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "pread") + public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "pread", required = 1, optional = 3) + public IRubyObject pread(ThreadContext context, IRubyObject[] args) { + Arity.checkArgumentCount(context, args, 3, 4); + + switch (args.length) { + case 3: + return pread(context, args[0], args[1], args[2]); + case 4: + return pread(context, args[0], args[1], args[2], args[3]); + } + + return context.nil; + } + + public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "write") + public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "write") + public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject length, IRubyObject offset) { + return context.nil; + } + + @JRubyMethod(name = "pwrite") + public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length) { + return context.nil; + } + + @JRubyMethod(name = "pwrite", required = 1, optional = 3) + public IRubyObject pwrite(ThreadContext context, IRubyObject[] args) { + Arity.checkArgumentCount(context, args, 3, 4); + + switch (args.length) { + case 3: + return pwrite(context, args[0], args[1], args[2]); + case 4: + return pwrite(context, args[0], args[1], args[2], args[3]); + } + + return context.nil; + } + + public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length, IRubyObject offset) { + return context.nil; + } + + private byte[] base; + private int size; + private int flags; +} diff --git a/core/src/main/java/org/jruby/RubyThread.java b/core/src/main/java/org/jruby/RubyThread.java index 35f425b16a5..49b3de386cd 100644 --- a/core/src/main/java/org/jruby/RubyThread.java +++ b/core/src/main/java/org/jruby/RubyThread.java @@ -64,7 +64,6 @@ import org.jruby.exceptions.RaiseException; import org.jruby.exceptions.ThreadKill; import org.jruby.exceptions.Unrescuable; -import org.jruby.ext.thread.Mutex; import org.jruby.internal.runtime.RubyNativeThread; import org.jruby.internal.runtime.RubyRunnable; import org.jruby.internal.runtime.ThreadLike; @@ -84,7 +83,6 @@ import org.jruby.runtime.backtrace.FrameType; import org.jruby.runtime.backtrace.RubyStackTraceElement; import org.jruby.runtime.builtin.IRubyObject; -import org.jruby.runtime.load.LoadService; import org.jruby.util.ByteList; import org.jruby.util.StringSupport; import org.jruby.util.io.BlockingIO; @@ -218,6 +216,9 @@ public enum Status { private volatile RubyThread fiberCurrentThread; + private IRubyObject scheduler; + private boolean blocking = true; + private static final AtomicIntegerFieldUpdater INTERRUPT_FLAG_UPDATER = AtomicIntegerFieldUpdater.newUpdater(RubyThread.class, "interruptFlag"); @@ -236,6 +237,7 @@ protected RubyThread(Ruby runtime, RubyClass type, boolean adopted) { finalResult = errorInfo = runtime.getNil(); reportOnException = runtime.isReportOnException(); + scheduler = runtime.getNil(); this.adopted = adopted; } @@ -2559,4 +2561,40 @@ public static IRubyObject uninterruptible(ThreadContext context, Sta state, f); } + + /** + * Set the scheduler for the current thread. + * + * MRI: rb_fiber_scheduler_set + */ + public IRubyObject setFiberScheduler(IRubyObject scheduler) { +// VM_ASSERT(ruby_thread_has_gvl_p()); + + scheduler.getClass(); // !null + + if (scheduler != null && !scheduler.isNil()) { + FiberScheduler.verifyInterface(scheduler); + } + + if (!this.scheduler.isNil()) { + FiberScheduler.close(getContext(), this.scheduler); + } + + this.scheduler = scheduler; + + return scheduler; + } + + public IRubyObject getScheduler() { + return scheduler; + } + + // MRI: rb_fiber_scheduler_current_for_threadptr, rb_fiber_scheduler_current + public IRubyObject getSchedulerCurrent() { + if (!blocking) { + return scheduler; + } + + return getRuntime().getNil(); + } } diff --git a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java index 36b803899dc..4993e34b1cc 100644 --- a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java +++ b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java @@ -635,6 +635,40 @@ public IRubyObject backtrace_locations(ThreadContext context, IRubyObject level, return threadFiber.thread.backtrace_locations(context, level, length); } + // MRI: rb_fiber_s_schedule_kw and rb_fiber_s_schedule, kw passes on context + @JRubyMethod(name = "schedule", meta = true, rest = true, keywords = true) + public static IRubyObject schedule(ThreadContext context, IRubyObject self, IRubyObject[] args, Block block) { + RubyThread thread = context.getFiberCurrentThread(); + IRubyObject scheduler = thread.getScheduler(); + IRubyObject fiber = context.nil; + + if (!scheduler.isNil()) { + fiber = scheduler.callMethod(context, "fiber", args, block); + } else { + throw context.runtime.newRuntimeError("No scheduler is available!"); + } + + return fiber; + } + + // MRI: rb_fiber_s_scheduler + @JRubyMethod(name = "scheduler", meta = true) + public static IRubyObject get_scheduler(ThreadContext context, IRubyObject self) { + return context.getFiberCurrentThread().getScheduler(); + } + + // MRI: rb_fiber_current_scheduler + @JRubyMethod(name = "current_scheduler", meta = true) + public static IRubyObject current_scheduler(ThreadContext context, IRubyObject self) { + return context.getFiberCurrentThread().getSchedulerCurrent(); + } + + // MRI: rb_fiber_set_scheduler + @JRubyMethod(name = "set_scheduler", meta = true) + public static IRubyObject set_scheduler(ThreadContext context, IRubyObject self, IRubyObject scheduler) { + return context.getFiberCurrentThread().setFiberScheduler(scheduler); + } + public FiberData getData() { return data; } diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index aad2c7d16c4..9c6f628971b 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -9,7 +9,6 @@ import java.nio.channels.SeekableByteChannel; import java.nio.channels.SelectableChannel; import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.nio.channels.WritableByteChannel; import java.util.HashSet; @@ -28,12 +27,14 @@ import org.jruby.Finalizable; import org.jruby.Ruby; import org.jruby.RubyArgsFile; +import org.jruby.RubyBasicObject; import org.jruby.RubyBignum; import org.jruby.RubyEncoding; import org.jruby.RubyException; import org.jruby.RubyFixnum; import org.jruby.RubyIO; import org.jruby.RubyNumeric; +import org.jruby.FiberScheduler; import org.jruby.RubyString; import org.jruby.RubyThread; import org.jruby.exceptions.RaiseException; @@ -457,6 +458,8 @@ public int io_fflush(ThreadContext context) { // rb_io_wait_writable public boolean waitWritable(ThreadContext context, long timeout) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + boolean locked = lock(); try { if (posix.getErrno() == null) return false; @@ -470,6 +473,10 @@ public boolean waitWritable(ThreadContext context, long timeout) { return true; case EAGAIN: case EWOULDBLOCK: + if (!scheduler.isNil()) { + return FiberScheduler.ioWaitWritable(context, scheduler, RubyIO.newIO(context.runtime, channel())).isTrue(); + } + ready(runtime, context.getThread(), SelectExecutor.WRITE_CONNECT_OPS, timeout); return true; default: @@ -487,6 +494,8 @@ public boolean waitWritable(ThreadContext context) { // rb_io_wait_readable public boolean waitReadable(ThreadContext context, long timeout) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + boolean locked = lock(); try { if (posix.getErrno() == null) return false; @@ -500,6 +509,10 @@ public boolean waitReadable(ThreadContext context, long timeout) { return true; case EAGAIN: case EWOULDBLOCK: + if (!scheduler.isNil()) { + return FiberScheduler.ioWaitReadable(context, scheduler, RubyIO.newIO(context.runtime, channel())).isTrue(); + } + ready(runtime, context.getThread(), SelectionKey.OP_READ, timeout); return true; default: @@ -1371,8 +1384,16 @@ public void wakeup(RubyThread thread, OpenFile data) { } }; - // rb_read_internal + // rb_read_internal, rb_io_read_memory public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, byte[] bufBytes, int buf, int count) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + + if (result != RubyBasicObject.UNDEF) { + FiberScheduler.resultApply(context, result); + } + } // if we can do selection and this is not a non-blocking call, do selection /* @@ -2312,8 +2333,17 @@ static int binwriteString(OpenFile fptr, byte[] bytes, int start, int length) { return fptr.writeInternal2(fptr.fd, bytes, start, l); } - // rb_write_internal + // rb_write_internal, rb_io_write_memory public static int writeInternal(ThreadContext context, OpenFile fptr, byte[] bufBytes, int buf, int count) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + + if (result != RubyBasicObject.UNDEF) { + FiberScheduler.resultApply(context, result); + } + } + try { return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, WRITE_TASK); } catch (InterruptedException ie) { From 3ba5662b8a2d71b8e1c8c591cede0bee24735f0a Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 15:58:59 -0500 Subject: [PATCH 08/56] Start filling in IO::Buffer logic --- core/src/main/java/org/jruby/Ruby.java | 40 +- .../src/main/java/org/jruby/RubyIOBuffer.java | 932 ++++++++++++++++-- .../main/java/org/jruby/runtime/Helpers.java | 12 + 3 files changed, 911 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/org/jruby/Ruby.java b/core/src/main/java/org/jruby/Ruby.java index eaa8ebebe99..03dbc4da8dc 100644 --- a/core/src/main/java/org/jruby/Ruby.java +++ b/core/src/main/java/org/jruby/Ruby.java @@ -1718,10 +1718,10 @@ private void initExceptions() { RubyClass runtimeError = this.runtimeError; ObjectAllocator runtimeErrorAllocator = runtimeError.getAllocator(); bufferLockedError = ioBufferClass.defineClassUnder("LockedError", runtimeError, runtimeErrorAllocator); - ioBufferClass.defineClassUnder("AllocationError", runtimeError, runtimeErrorAllocator); - ioBufferClass.defineClassUnder("AccessError", runtimeError, runtimeErrorAllocator); - ioBufferClass.defineClassUnder("InvalidatedError", runtimeError, runtimeErrorAllocator); - ioBufferClass.defineClassUnder("MaskError", runtimeError, runtimeErrorAllocator); + bufferAllocationError = ioBufferClass.defineClassUnder("AllocationError", runtimeError, runtimeErrorAllocator); + bufferAccessError = ioBufferClass.defineClassUnder("AccessError", runtimeError, runtimeErrorAllocator); + bufferInvalidatedError = ioBufferClass.defineClassUnder("InvalidatedError", runtimeError, runtimeErrorAllocator); + bufferMaskError = ioBufferClass.defineClassUnder("MaskError", runtimeError, runtimeErrorAllocator); initErrno(); @@ -2502,6 +2502,22 @@ public RubyClass getBufferLockedError() { return bufferLockedError; } + public RubyClass getBufferAllocationError() { + return bufferAllocationError; + } + + public RubyClass getBufferAccessError() { + return bufferAccessError; + } + + public RubyClass getBufferInvalidatedError() { + return bufferInvalidatedError; + } + + public RubyClass getBufferMaskError() { + return bufferMaskError; + } + @Deprecated RubyRandom.RandomType defaultRand; @@ -4267,6 +4283,22 @@ public RaiseException newBufferLockedError(String message) { return newRaiseException(getBufferLockedError(), message); } + public RaiseException newBufferAllocationError(String message) { + return newRaiseException(getBufferAllocationError(), message); + } + + public RaiseException newBufferAccessError(String message) { + return newRaiseException(getBufferAccessError(), message); + } + + public RaiseException newBufferInvalidatedError(String message) { + return newRaiseException(getBufferInvalidatedError(), message); + } + + public RaiseException newBufferMaskError(String message) { + return newRaiseException(getBufferMaskError(), message); + } + /** * Construct a new RaiseException wrapping a new Ruby exception object appropriate to the given exception class. * diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 47ca2969f35..d3ec4848f61 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1,14 +1,31 @@ package org.jruby; +import jnr.ffi.NativeType; import jnr.ffi.Platform; +import jnr.ffi.Runtime; +import jnr.ffi.Type; import org.jruby.anno.JRubyConstant; import org.jruby.anno.JRubyMethod; import org.jruby.runtime.Arity; import org.jruby.runtime.Block; +import org.jruby.runtime.Helpers; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.func.ObjectLongFunction; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.function.LongUnaryOperator; +import java.util.function.ToLongFunction; + +import static org.jruby.RubyBoolean.newBoolean; public class RubyIOBuffer extends RubyObject { + + public static final Runtime FFI_RUNTIME = Runtime.getSystemRuntime(); + public static RubyClass createIOBufferClass(Ruby runtime) { RubyClass IOBuffer = runtime.getIO().defineClassUnder("Buffer", runtime.getObject(), RubyIOBuffer::new); @@ -47,7 +64,7 @@ public static RubyClass createIOBufferClass(Ruby runtime) { @JRubyConstant public static final int NETWORK_ENDIAN = BIG_ENDIAN; - public static RubyIOBuffer newBuffer(Ruby runtime, byte[] base, int size, int flags) { + public static RubyIOBuffer newBuffer(Ruby runtime, ByteBuffer base, int size, int flags) { return new RubyIOBuffer(runtime, runtime.getIO(), base, size, flags); } @@ -55,7 +72,7 @@ public RubyIOBuffer(Ruby runtime, RubyClass metaClass) { super(runtime, metaClass); } - public RubyIOBuffer(Ruby runtime, RubyClass metaClass, byte[] base, int size, int flags) { + public RubyIOBuffer(Ruby runtime, RubyClass metaClass, ByteBuffer base, int size, int flags) { super(runtime, metaClass); this.base = base; @@ -70,17 +87,60 @@ public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyOb @JRubyMethod(name = "initialize") public IRubyObject initialize(ThreadContext context) { - return context.nil; + return initialize(context, DEFAULT_SIZE); } @JRubyMethod(name = "initialize") public IRubyObject initialize(ThreadContext context, IRubyObject size) { - return context.nil; + return initialize(context, size.convertToInteger().getIntValue()); } @JRubyMethod(name = "initialize") public IRubyObject initialize(ThreadContext context, IRubyObject size, IRubyObject flags) { - return context.nil; + return initialize(context, size.convertToInteger().getIntValue(), flags.convertToInteger().getIntValue(), context.nil); + } + + public IRubyObject initialize(ThreadContext context, int size) { + return initialize(context, size, flagsForSize(size), context.nil); + } + + // MRI: io_buffer_initialize + public void initialize(ThreadContext context, byte[] baseBytes, int size, int flags, IRubyObject source) { + ByteBuffer base = null; + + if (baseBytes != null) { + // If we are provided a pointer, we use it. + base = ByteBuffer.wrap(baseBytes); + } else if (size != 0) { + // If we are provided a non-zero size, we allocate it: + if ((flags & INTERNAL) == INTERNAL) { + base = ByteBuffer.allocate(size); + } else if ((flags & MAPPED) == MAPPED) { + // no support for SHARED, PRIVATE yet + base = ByteBuffer.allocateDirect(size); + } + + if (base == null) { + throw context.runtime.newBufferAllocationError("Could not allocate buffer!"); + } + } else { + // Otherwise we don't do anything. + return; + } + + this.base = base; + this.size = size; + this.flags = flags; + this.source = source; + } + + // MRI: io_flags_for_size + private static int flagsForSize(int size) { + if (size >= PAGE_SIZE) { + return MAPPED; + } + + return INTERNAL; } @JRubyMethod(name = "initialize_copy") @@ -105,7 +165,7 @@ public IRubyObject to_s(ThreadContext context) { @JRubyMethod(name = "size") public IRubyObject size(ThreadContext context) { - return context.nil; + return context.runtime.newFixnum(size); } @JRubyMethod(name = "valid?") @@ -115,58 +175,110 @@ public IRubyObject valid_p(ThreadContext context) { @JRubyMethod(name = "transfer") public IRubyObject transfer(ThreadContext context) { - return context.nil; + if (isLocked()) { + throw context.runtime.newBufferLockedError("Cannot transfer ownership of locked buffer!"); + } + + RubyIOBuffer instance = new RubyIOBuffer(context.runtime, getMetaClass()); + + instance.base = base; + instance.size = size; + instance.flags = flags; + instance.source = source; + + zero(context); + + return instance; + } + + private void zero(ThreadContext context) { + base = null; + size = 0; + source = context.nil; } @JRubyMethod(name = "null?") public IRubyObject null_p(ThreadContext context) { - return context.nil; + return newBoolean(context, base == null); } @JRubyMethod(name = "empty?") public IRubyObject empty_p(ThreadContext context) { - return context.nil; + return newBoolean(context, size == 0); } @JRubyMethod(name = "external?") public IRubyObject external_p(ThreadContext context) { - return context.nil; + return newBoolean(context, isExternal()); + } + + private boolean isExternal() { + return (flags & EXTERNAL) == EXTERNAL; } @JRubyMethod(name = "internal?") public IRubyObject internal_p(ThreadContext context) { - return context.nil; + return newBoolean(context, isInternal()); + } + + private boolean isInternal() { + return (flags & INTERNAL) == INTERNAL; } @JRubyMethod(name = "mapped?") public IRubyObject mapped_p(ThreadContext context) { - return context.nil; + return newBoolean(context, isMapped()); + } + + private boolean isMapped() { + return (flags & MAPPED) == MAPPED; } @JRubyMethod(name = "shared?") public IRubyObject shared_p(ThreadContext context) { - return context.nil; + // no support for shared yet + return newBoolean(context, false); } @JRubyMethod(name = "locked?") public IRubyObject locked_p(ThreadContext context) { - return context.nil; + return newBoolean(context, isLocked()); + } + + private boolean isLocked() { + return (flags & LOCKED) == LOCKED; } @JRubyMethod(name = "readonly?") public IRubyObject readonly_p(ThreadContext context) { - return context.nil; + return newBoolean(context, isReadonly()); + } + + private boolean isReadonly() { + return (flags & READONLY) == READONLY; } @JRubyMethod(name = "locked") public IRubyObject locked(ThreadContext context, Block block) { - return context.nil; + checkLocked(context); + + flags |= LOCKED; + + IRubyObject result = block.yield(context, this); + + flags &= ~LOCKED; + + return result; } - public IRubyObject lock(ThreadContext context) { - if ((flags & LOCKED) == LOCKED) { + private void checkLocked(ThreadContext context) { + if (isLocked()) { throw context.runtime.newBufferLockedError("Buffer already locked!"); } + } + + public IRubyObject lock(ThreadContext context) { + checkLocked(context); flags |= LOCKED; @@ -183,24 +295,70 @@ public IRubyObject unlock(ThreadContext context) { return this; } + private boolean tryUnlock() { + if (isLocked()) { + flags &= ~LOCKED; + return true; + } + + return false; + } + @JRubyMethod(name = "slice") public IRubyObject slice(ThreadContext context) { - return context.nil; + return slice(context, 0, size); } @JRubyMethod(name = "slice") - public IRubyObject slice(ThreadContext context, IRubyObject offset) { - return context.nil; + public IRubyObject slice(ThreadContext context, IRubyObject _offset) { + int offset = RubyNumeric.num2int(_offset); + + if (offset < 0) { + throw context.runtime.newArgumentError("Offset can't be negative!"); + } + + return slice(context, offset, size - offset); } @JRubyMethod(name = "slice") - public IRubyObject slice(ThreadContext context, IRubyObject offset, IRubyObject length) { - return context.nil; + public IRubyObject slice(ThreadContext context, IRubyObject _offset, IRubyObject _length) { + int offset = RubyNumeric.num2int(_offset); + + if (offset < 0) { + throw context.runtime.newArgumentError("Offset can't be negative!"); + } + + int length = RubyNumeric.num2int(_length); + + if (length < 0) { + throw context.runtime.newArgumentError("Length can't be negative!"); + } + + return slice(context, offset, length); + } + + // MRI: rb_io_buffer_slice + public IRubyObject slice(ThreadContext context, int offset, int length) { + validateRange(context, offset, length); + + // gross, but slice(int, int) is 13+ + base.position(offset); + base.limit(offset + length); + ByteBuffer slice = base.slice(); + base.clear(); + + return newBuffer(context.runtime, slice, length, flags); + } + + private void validateRange(ThreadContext context, int offset, int length) { + if (offset + length > size) { + throw context.runtime.newArgumentError("Specified offset+length exceeds data size!"); + } } @JRubyMethod(name = "<=>") public IRubyObject op_cmp(ThreadContext context, IRubyObject other) { - return context.nil; + return context.runtime.newFixnum(base.compareTo(((RubyIOBuffer) other).base)); } @JRubyMethod(name = "resize") @@ -208,117 +366,642 @@ public IRubyObject resize(ThreadContext context, IRubyObject size) { return context.nil; } + // MRI: rb_io_buffer_resize + public void resize(ThreadContext context, int size) { + if (isLocked()) { + throw context.runtime.newBufferLockedError("Cannot resize locked buffer!"); + } + + if (this.base == null) { + initialize(context, null, size, flagsForSize(size), context.nil); + return; + } + + if (isExternal()) { + throw context.runtime.newBufferAccessError("Cannot resize external buffer!"); + } + + // no special behavior for isInternal=true since we do not control the internals of ByteBuffers. + + ByteBuffer newBase = this.base.isDirect() ? ByteBuffer.allocateDirect(size) : ByteBuffer.allocate(size); + newBase.put(this.base); + + this.base = newBase; + this.size = size; + } + + // MRI: io_buffer_clear @JRubyMethod(name = "clear") public IRubyObject clear(ThreadContext context) { - return context.nil; + return clear(context, 0, 0, size); } @JRubyMethod(name = "clear") public IRubyObject clear(ThreadContext context, IRubyObject value) { - return context.nil; + return clear(context, RubyNumeric.num2int(value), 0, size); } @JRubyMethod(name = "clear") - public IRubyObject clear(ThreadContext context, IRubyObject value, IRubyObject offset) { - return context.nil; + public IRubyObject clear(ThreadContext context, IRubyObject _value, IRubyObject _offset) { + int value = RubyNumeric.num2int(_value); + int offset = RubyNumeric.num2int(_offset); + return clear(context, value, offset, size - offset); } @JRubyMethod(name = "clear") - public IRubyObject clear(ThreadContext context, IRubyObject value, IRubyObject offset, IRubyObject length) { - return context.nil; + public IRubyObject clear(ThreadContext context, IRubyObject _value, IRubyObject _offset, IRubyObject _length) { + int value = RubyNumeric.num2int(_value); + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + return clear(context, value, offset, length); + } + + // MRI: rb_io_buffer_clear + private IRubyObject clear(ThreadContext context, int value, int offset, int length) { + ByteBuffer buffer = getBufferForWriting(context); + + if (offset + length > size) { + throw context.runtime.newArgumentError("The given offset + length out of bounds!"); + } + + if (buffer.hasArray()) { + Arrays.fill(buffer.array(), offset, offset + length, (byte) value); + } + + return this; + } + + private ByteBuffer getBufferForWriting(ThreadContext context) { + if (isReadonly()) { + throw context.runtime.newBufferAccessError("Buffer is not writable!"); + } + + // TODO: validate our buffer + + if (base != null) { + return base; + } + + throw context.runtime.newBufferAllocationError("The buffer is not allocated!"); + } + + private ByteBuffer getBufferForReading(ThreadContext context) { + // TODO: validate our buffer + + if (base != null) { + return base; + } + + throw context.runtime.newBufferAllocationError("The buffer is not allocated!"); } @JRubyMethod(name = "free") public IRubyObject free(ThreadContext context) { - return context.nil; + if (isLocked()) { + throw context.runtime.newBufferLockedError("Buffer is locked!"); + } + + freeInternal(context); + + return this; + } + + private boolean freeInternal(ThreadContext context) { + if (this.base != null) { + // No special handling for internal yet + + // No special handling for mapped yet + + // We can only dereference and allow GC to clean it up + this.base = null; + this.size = 0; + this.flags = 0; + this.source = context.nil; + + return true; + } + + return false; } @JRubyMethod(name = "size_of") public static IRubyObject size_of(ThreadContext context, IRubyObject self, IRubyObject dataType) { - return context.nil; + if (dataType instanceof RubyArray) { + long total = 0; + RubyArray array = (RubyArray) dataType; + int size = array.size(); + for (int i = 0; i < size; i++) { + IRubyObject elt = array.eltOk(i); + total += getDataType(elt).type.size(); + } + } + + return RubyFixnum.newFixnum(context.runtime, getDataType(dataType).type.size()); + } + + private boolean isBigEndian() { + return (flags & BIG_ENDIAN) == BIG_ENDIAN; + } + + private boolean isLittleEndian() { + return (flags & LITTLE_ENDIAN) == LITTLE_ENDIAN; + } + + private boolean isHostEndian() { + return (flags & (BIG_ENDIAN | LITTLE_ENDIAN)) == HOST_ENDIAN; + } + + private static DataType getDataType(IRubyObject dataType) { + return DataType.valueOf(RubySymbol.objectToSymbolString(dataType)); + } + + private static byte readByte(ThreadContext context, ByteBuffer buffer, int offset) { + return buffer.get(offset); + } + + private static int readUnsignedByte(ThreadContext context, ByteBuffer buffer, int offset) { + return Byte.toUnsignedInt(buffer.get(offset)); + } + + private static void writeByte(ThreadContext context, ByteBuffer buffer, int offset, byte value) { + buffer.put(offset, (byte) value); + } + + private static void writeUnsignedByte(ThreadContext context, ByteBuffer buffer, int offset, int value) { + buffer.put(offset, (byte) value); + } + + private static short readShort(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + short s = buffer.getShort(offset); + + if (order == ByteOrder.BIG_ENDIAN) return s; + + return Short.reverseBytes(s); + } + + private static int readUnsignedShort(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + return Short.toUnsignedInt(readShort(context, buffer, offset, order)); + } + + private static void writeShort(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, short value) { + if (order == ByteOrder.BIG_ENDIAN) buffer.putShort(offset, value); + + buffer.putShort(offset, Short.reverseBytes(value)); + } + + private static void writeUnsignedShort(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, int value) { + writeShort(context, buffer, offset, order, (short) value); + } + + private static int readInt(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + int i = buffer.getInt(offset); + + if (order == ByteOrder.BIG_ENDIAN) return i; + + return Integer.reverseBytes(i); + } + + private static long readUnsignedInt(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + return Integer.toUnsignedLong(readInt(context, buffer, offset, order)); + } + + private static void writeInt(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, int value) { + if (order == ByteOrder.BIG_ENDIAN) buffer.putInt(offset, value); + + buffer.putInt(offset, Integer.reverseBytes(value)); + } + + private static void writeUnsignedInt(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, long value) { + writeInt(context, buffer, offset, order, (int) value); + } + + private static long readLong(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + long l = buffer.getLong(offset); + + if (order == ByteOrder.BIG_ENDIAN) return l; + + return Long.reverseBytes(l); + } + + private static BigInteger readUnsignedLong(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + long l = readLong(context, buffer, offset, order); + if (l > 0L) return BigInteger.valueOf(l); + + byte[] bytes = new byte[8]; + for (int i = 7; i >= 0; i--) { + bytes[i] = (byte)(l & 0xFF); + l >>= 8; + } + + return new BigInteger(1, bytes); + } + + private static void writeLong(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, long value) { + if (order == ByteOrder.BIG_ENDIAN) buffer.putLong(offset, value); + + buffer.putLong(offset, Long.reverseBytes(value)); + } + + private static void writeUnsignedLong(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, long value) { + writeLong(context, buffer, offset, order, value); + } + + private static float readFloat(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + float f = buffer.getFloat(offset); + + if (order == ByteOrder.BIG_ENDIAN) return f; + + return Float.intBitsToFloat(Integer.reverseBytes(Float.floatToIntBits(f))); + } + + private static void writeFloat(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, float value) { + if (order == ByteOrder.BIG_ENDIAN) buffer.putFloat(offset, value); + + buffer.putFloat(offset, Float.intBitsToFloat(Integer.reverseBytes(Float.floatToIntBits(value)))); + } + + private static double readDouble(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order) { + double f = buffer.getDouble(offset); + + if (order == ByteOrder.BIG_ENDIAN) return f; + + return Double.longBitsToDouble(Long.reverseBytes(Double.doubleToLongBits(f))); + } + + private static void writeDouble(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, double value) { + if (order == ByteOrder.BIG_ENDIAN) buffer.putDouble(offset, value); + + buffer.putDouble(offset, Double.longBitsToDouble(Long.reverseBytes(Double.doubleToLongBits(value)))); + } + + private static IRubyObject wrap(Ruby runtime, long value) { + return RubyFixnum.newFixnum(runtime, value); + } + + private static IRubyObject wrap(Ruby runtime, BigInteger value) { + return RubyBignum.newBignum(runtime, value); + } + + private static IRubyObject wrap(Ruby runtime, double value) { + return RubyFloat.newFloat(runtime, value); + } + + private static long unwrapLong(IRubyObject value) { + return value.convertToInteger().getLongValue(); + } + + private static double unwrapDouble(IRubyObject value) { + return value.convertToFloat().getDoubleValue(); + } + + private static long unwrapUnsignedLong(IRubyObject value) { + return RubyNumeric.num2ulong(value); } @JRubyMethod(name = "get_value") - public IRubyObject get_value(ThreadContext context, IRubyObject type, IRubyObject offset) { - return context.nil; + public IRubyObject get_value(ThreadContext context, IRubyObject type, IRubyObject _offset) { + ByteBuffer buffer = getBufferForReading(context); + + DataType dataType = getDataType(type); + int offset = RubyNumeric.num2int(_offset); + int size = this.size; + + return getValue(context, buffer, size, dataType, offset); + } + + private static IRubyObject getValue(ThreadContext context, ByteBuffer buffer, int size, DataType dataType, int offset) { + Ruby runtime = context.runtime; + + // TODO: validate size + + switch (dataType) { + case S8: + return wrap(runtime, readByte(context, buffer, offset)); + case U8: + return wrap(runtime, readUnsignedByte(context, buffer, offset)); + case u16: + return wrap(runtime, readUnsignedShort(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case U16: + return wrap(runtime, readUnsignedShort(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case s16: + return wrap(runtime, readShort(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case S16: + return wrap(runtime, readShort(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case u32: + return wrap(runtime, readUnsignedInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case U32: + return wrap(runtime, readUnsignedInt(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case s32: + return wrap(runtime, readInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case S32: + return wrap(runtime, readInt(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case u64: + return wrap(runtime, readUnsignedLong(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case U64: + return wrap(runtime, readUnsignedLong(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case s64: + return wrap(runtime, readLong(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case S64: + return wrap(runtime, readLong(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case f32: + return wrap(runtime, readFloat(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case F32: + return wrap(runtime, readFloat(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + case f64: + return wrap(runtime, readDouble(context, buffer, offset, ByteOrder.LITTLE_ENDIAN)); + case F64: + return wrap(runtime, readDouble(context, buffer, offset, ByteOrder.BIG_ENDIAN)); + } + + throw runtime.newArgumentError("Unknown data_type: " + dataType); // should never happen } @JRubyMethod(name = "get_values") - public IRubyObject get_values(ThreadContext context, IRubyObject data_types, IRubyObject offset) { - return context.nil; + public IRubyObject get_values(ThreadContext context, IRubyObject dataTypes, IRubyObject _offset) { + Ruby runtime = context.runtime; + + int offset = RubyNumeric.num2int(_offset); + + int size = this.size; + + ByteBuffer buffer = getBufferForReading(context); + + if (!(dataTypes instanceof RubyArray)) { + throw runtime.newArgumentError("Argument data_types should be an array!"); + } + + RubyArray dataTypesArray = (RubyArray) dataTypes; + int dataTypesSize = dataTypesArray.size(); + RubyArray values = RubyArray.newArray(runtime, dataTypesSize); + + for (long i = 0; i < dataTypesSize; i++) { + IRubyObject type = dataTypesArray.eltOk(i); + DataType dataType = getDataType(type); + + IRubyObject value = getValue(context, buffer, size, dataType, offset); + + offset += dataType.type.size(); + + values.push(value); + } + + return values; } @JRubyMethod(name = "each") - public IRubyObject each(ThreadContext context, IRubyObject dataType, Block block) { - return context.nil; + public IRubyObject each(ThreadContext context, IRubyObject _dataType, Block block) { + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", _dataType); + + ByteBuffer buffer = getBufferForReading(context); + DataType dataType = getDataType(_dataType); + + return each(context, buffer, dataType, 0, size, block); } @JRubyMethod(name = "each") - public IRubyObject each(ThreadContext context, IRubyObject dataType, IRubyObject offset, Block block) { - return context.nil; + public IRubyObject each(ThreadContext context, IRubyObject _dataType, IRubyObject _offset, Block block) { + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", Helpers.arrayOf(_dataType, _offset)); + + ByteBuffer buffer = getBufferForReading(context); + DataType dataType = getDataType(_dataType); + int offset = _offset.convertToInteger().getIntValue(); + + return each(context, buffer, dataType, offset, size - offset, block); } @JRubyMethod(name = "each") - public IRubyObject each(ThreadContext context, IRubyObject dataType, IRubyObject offset, IRubyObject count, Block block) { - return context.nil; + public IRubyObject each(ThreadContext context, IRubyObject _dataType, IRubyObject _offset, IRubyObject _count, Block block) { + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", Helpers.arrayOf(_dataType, _offset, _count)); + + ByteBuffer buffer = getBufferForReading(context); + DataType dataType = getDataType(_dataType); + int offset = _offset.convertToInteger().getIntValue(); + int count = _count.convertToInteger().getIntValue(); + + return each(context, buffer, dataType, offset, count, block); + } + + private IRubyObject each(ThreadContext context, ByteBuffer buffer, DataType dataType, int offset, int count, Block block) { + Ruby runtime = context.runtime; + + for (int i = 0 ; i < count; i++) { + int currentOffset = offset; + IRubyObject value = getValue(context, buffer, size, dataType, offset); + offset += dataType.type.size(); + block.yieldSpecific(context, RubyFixnum.newFixnum(runtime, currentOffset), value); + } + + return this; } @JRubyMethod(name = "values") - public IRubyObject values(ThreadContext context, IRubyObject dataType) { - return context.nil; + public IRubyObject values(ThreadContext context, IRubyObject _dataType) { + ByteBuffer buffer = getBufferForReading(context); + DataType dataType = getDataType(_dataType); + + return values(context, buffer, dataType, 0, size); } @JRubyMethod(name = "values") - public IRubyObject values(ThreadContext context, IRubyObject dataType, IRubyObject offset) { - return context.nil; + public IRubyObject values(ThreadContext context, IRubyObject _dataType, IRubyObject _offset) { + ByteBuffer buffer = getBufferForReading(context); + DataType dataType = getDataType(_dataType); + int offset = _offset.convertToInteger().getIntValue(); + + return values(context, buffer, dataType, offset, size - offset); } @JRubyMethod(name = "values") - public IRubyObject values(ThreadContext context, IRubyObject dataType, IRubyObject offset, IRubyObject count) { - return context.nil; + public IRubyObject values(ThreadContext context, IRubyObject _dataType, IRubyObject _offset, IRubyObject _count) { + ByteBuffer buffer = getBufferForReading(context); + DataType dataType = getDataType(_dataType); + int offset = _offset.convertToInteger().getIntValue(); + int count = _count.convertToInteger().getIntValue(); + + return values(context, buffer, dataType, offset, count); + } + + private RubyArray values(ThreadContext context, ByteBuffer buffer, DataType dataType, int offset, int count) { + Ruby runtime = context.runtime; + + RubyArray values = RubyArray.newArray(runtime, count); + + for (int i = 0 ; i < count; i++) { + int currentOffset = offset; + IRubyObject value = getValue(context, buffer, size, dataType, offset); + offset += dataType.type.size(); + values.push(value); + } + + return values; } @JRubyMethod(name = "each_byte") public IRubyObject each_byte(ThreadContext context, Block block) { - return context.nil; + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each"); + + ByteBuffer buffer = getBufferForReading(context); + + return eachByte(context, buffer, 0, size, block); } @JRubyMethod(name = "each_byte") - public IRubyObject each_byte(ThreadContext context, IRubyObject offset, Block block) { - return context.nil; + public IRubyObject each_byte(ThreadContext context, IRubyObject _offset, Block block) { + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", Helpers.arrayOf(_offset)); + + ByteBuffer buffer = getBufferForReading(context); + int offset = _offset.convertToInteger().getIntValue(); + + return eachByte(context, buffer, offset, size - offset, block); } @JRubyMethod(name = "each_byte") - public IRubyObject each_byte(ThreadContext context, IRubyObject offset, IRubyObject count, Block block) { - return context.nil; + public IRubyObject each_byte(ThreadContext context, IRubyObject _offset, IRubyObject _count, Block block) { + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", Helpers.arrayOf(_offset, _count)); + + ByteBuffer buffer = getBufferForReading(context); + int offset = _offset.convertToInteger().getIntValue(); + int count = _count.convertToInteger().getIntValue(); + + return eachByte(context, buffer, offset, count, block); + } + + private IRubyObject eachByte(ThreadContext context, ByteBuffer buffer, int offset, int count, Block block) { + Ruby runtime = context.runtime; + + for (int i = 0 ; i < count; i++) { + IRubyObject value = wrap(runtime, readByte(context, buffer, offset)); + block.yieldSpecific(context, value); + } + + return this; + } + + private static void setValue(ThreadContext context, ByteBuffer buffer, int size, DataType dataType, int offset, IRubyObject value) { + // TODO: validate size + + switch (dataType) { + case S8: + writeByte(context, buffer, offset, (byte) unwrapLong(value)); + case U8: + writeUnsignedByte(context, buffer, offset, (int) unwrapLong(value)); + case u16: + writeUnsignedShort(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (int) unwrapLong(value)); + case U16: + writeUnsignedShort(context, buffer, offset, ByteOrder.BIG_ENDIAN, (int) unwrapLong(value)); + case s16: + writeShort(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (short) unwrapLong(value)); + case S16: + writeShort(context, buffer, offset, ByteOrder.BIG_ENDIAN, (short) unwrapLong(value)); + case u32: + writeUnsignedInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (long) unwrapLong(value)); + case U32: + writeUnsignedInt(context, buffer, offset, ByteOrder.BIG_ENDIAN, (long) unwrapLong(value)); + case s32: + writeInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (int) unwrapLong(value)); + case S32: + writeInt(context, buffer, offset, ByteOrder.BIG_ENDIAN, (int) unwrapLong(value)); + case u64: + writeUnsignedLong(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapUnsignedLong(value)); + case U64: + writeUnsignedLong(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapUnsignedLong(value)); + case s64: + writeLong(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapLong(value)); + case S64: + writeLong(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapLong(value)); + case f32: + writeFloat(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (float) unwrapDouble(value)); + case F32: + writeFloat(context, buffer, offset, ByteOrder.BIG_ENDIAN, (float) unwrapDouble(value)); + case f64: + writeDouble(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapDouble(value)); + case F64: + writeDouble(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapDouble(value)); + } + + throw context.runtime.newArgumentError("Unknown data_type: " + dataType); // should never happen } @JRubyMethod(name = "set_value") - public IRubyObject set_value(ThreadContext context, IRubyObject dataType, IRubyObject offset, IRubyObject value) { - return context.nil; + public IRubyObject set_value(ThreadContext context, IRubyObject _dataType, IRubyObject _offset, IRubyObject _value) { + ByteBuffer buffer = getBufferForWriting(context); + + DataType dataType = getDataType(_dataType); + int offset = RubyNumeric.num2int(_offset); + int size = this.size; + + setValue(context, buffer, size, dataType, offset, _value); + + return RubyFixnum.newFixnum(context.runtime, offset + dataType.type.size()); } @JRubyMethod(name = "set_values") - public IRubyObject set_values(ThreadContext context, IRubyObject dataTypes, IRubyObject offset, IRubyObject values) { - return context.nil; + public IRubyObject set_values(ThreadContext context, IRubyObject _dataTypes, IRubyObject _offset, IRubyObject _values) { + Ruby runtime = context.runtime; + + int offset = RubyNumeric.num2int(_offset); + + int size = this.size; + + ByteBuffer buffer = getBufferForWriting(context); + + if (!(_dataTypes instanceof RubyArray)) { + throw runtime.newArgumentError("Argument data_types should be an array!"); + } + RubyArray dataTypes = (RubyArray) _dataTypes; + + if (!(_values instanceof RubyArray)) { + throw runtime.newArgumentError("Argument values should be an array!"); + } + RubyArray values = (RubyArray) _values; + + if (dataTypes.size() != values.size()) { + throw runtime.newArgumentError("Argument data_types and values should have the same length!"); + } + + int dataTypesSize = dataTypes.size(); + + for (long i = 0; i < dataTypesSize; i++) { + IRubyObject type = dataTypes.eltOk(i); + DataType dataType = getDataType(type); + + IRubyObject value = values.eltOk(i); + + setValue(context, buffer, size, dataType, offset, value); + + offset += dataType.type.size(); + } + + return RubyFixnum.newFixnum(runtime, offset); } @JRubyMethod(name = "copy") public IRubyObject copy(ThreadContext context, IRubyObject source) { - return context.nil; + RubyIOBuffer sourceBuffer = (RubyIOBuffer) source; + + return copy(context, sourceBuffer, 0, sourceBuffer.size, 0); } @JRubyMethod(name = "copy") - public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject offset) { - return context.nil; + public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject _offset) { + RubyIOBuffer sourceBuffer = (RubyIOBuffer) source; + + int offset = RubyNumeric.num2int(_offset); + + return copy(context, sourceBuffer, offset, sourceBuffer.size, 0); } @JRubyMethod(name = "copy") - public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject offset, IRubyObject length) { - return context.nil; + public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject _offset, IRubyObject _length) { + RubyIOBuffer sourceBuffer = (RubyIOBuffer) source; + + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + + return copy(context, sourceBuffer, offset, length, 0); } - @JRubyMethod(name = "copy", required = 1, optional = 3) + @JRubyMethod(name = "copy", required = 1, optional = 3, checkArity = false) public IRubyObject copy(ThreadContext context, IRubyObject[] args) { Arity.checkArgumentCount(context, args, 1, 3); @@ -336,8 +1019,35 @@ public IRubyObject copy(ThreadContext context, IRubyObject[] args) { return context.nil; } - public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject offset, IRubyObject length, IRubyObject sourceOffset) { - return context.nil; + public IRubyObject copy(ThreadContext context, IRubyObject source, IRubyObject _offset, IRubyObject _length, IRubyObject _sourceOffset) { + RubyIOBuffer sourceBuffer = (RubyIOBuffer) source; + + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + int sourceOffset = RubyNumeric.num2int(_sourceOffset); + + return copy(context, sourceBuffer, offset, length, sourceOffset); + } + + public IRubyObject copy(ThreadContext context, RubyIOBuffer source, int offset, int length, int sourceOffset) { + if (sourceOffset > length) { + throw context.runtime.newArgumentError("The given source offset is bigger than the source itself!"); + } + + ByteBuffer sourceBuffer = source.getBufferForReading(context); + + bufferCopy(context, offset, sourceBuffer, sourceOffset, source.size, length); + + return RubyFixnum.newFixnum(context.runtime, length); + } + + private void bufferCopy(ThreadContext context, int offset, ByteBuffer sourceBuffer, int sourceOffset, int sourceSize, int length) { + ByteBuffer destBuffer = getBufferForWriting(context); + + sourceBuffer.position(sourceOffset); + sourceBuffer.limit(sourceOffset + length); + destBuffer.put(sourceBuffer); + sourceBuffer.clear(); } @JRubyMethod(name = "get_string") @@ -370,7 +1080,7 @@ public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyOb return context.nil; } - @JRubyMethod(name = "set_string", required = 1, optional = 3) + @JRubyMethod(name = "set_string", required = 1, optional = 3, checkArity = false) public IRubyObject set_string(ThreadContext context, IRubyObject[] args) { Arity.checkArgumentCount(context, args, 1, 3); @@ -427,7 +1137,7 @@ public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject from return context.nil; } - @JRubyMethod(name = "pread", required = 1, optional = 3) + @JRubyMethod(name = "pread", required = 1, optional = 3, checkArity = false) public IRubyObject pread(ThreadContext context, IRubyObject[] args) { Arity.checkArgumentCount(context, args, 3, 4); @@ -460,7 +1170,7 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject fro return context.nil; } - @JRubyMethod(name = "pwrite", required = 1, optional = 3) + @JRubyMethod(name = "pwrite", required = 1, optional = 3, checkArity = false) public IRubyObject pwrite(ThreadContext context, IRubyObject[] args) { Arity.checkArgumentCount(context, args, 3, 4); @@ -478,7 +1188,91 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject fro return context.nil; } - private byte[] base; + enum DataType { + U8(NativeType.UCHAR, BIG_ENDIAN, DataType::ubyteToNum, RubyNumeric::num2ulong, LongUnaryOperator.identity()), + S8(NativeType.SCHAR, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2long, LongUnaryOperator.identity()), + u16(NativeType.USHORT, LITTLE_ENDIAN, DataType::ushortToNum, RubyNumeric::num2ulong, DataType::swapAsShort), + U16(NativeType.USHORT, BIG_ENDIAN, DataType::ushortToNum, RubyNumeric::num2ulong, DataType::swapAsShort), + s16(NativeType.SSHORT, LITTLE_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsShort), + S16(NativeType.SSHORT, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsShort), + u32(NativeType.UINT, LITTLE_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsInt), + U32(NativeType.UINT, BIG_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsInt), + s32(NativeType.SINT, LITTLE_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsInt), + S32(NativeType.SINT, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsInt), + u64(NativeType.ULONG, LITTLE_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsLong), + U64(NativeType.ULONG, BIG_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsLong), + s64(NativeType.SLONG, LITTLE_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsLong), + S64(NativeType.SLONG, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsLong), + f32(NativeType.FLOAT,LITTLE_ENDIAN, RubyFloat::newFloat, DataType::num2FloatBits, DataType::swapAsInt), + F32(NativeType.FLOAT, BIG_ENDIAN, RubyFloat::newFloat, DataType::num2FloatBits, DataType::swapAsInt), + f64(NativeType.DOUBLE, LITTLE_ENDIAN, RubyFloat::newFloat, DataType::num2DoubleBits, DataType::swapAsLong), + F64(NativeType.DOUBLE, BIG_ENDIAN, RubyFloat::newFloat, DataType::num2DoubleBits, DataType::swapAsLong); + + DataType(NativeType type, int endian, ObjectLongFunction toNum, ToLongFunction fromNum, LongUnaryOperator swap, Function) { + this.type = FFI_RUNTIME.findType(type); + this.endian = endian; + this.toNum = toNum; + this.fromNum = fromNum; + this.swap = swap; + } + + private final Type type; + private final int endian; + private final ObjectLongFunction toNum; + private final ToLongFunction fromNum; + private final LongUnaryOperator swap; + + private static IRubyObject uintToNum(Ruby r, long l) { + return RubyFixnum.newFixnum(r, Integer.toUnsignedLong((int) l)); + } + + private static IRubyObject ushortToNum(Ruby r, long l) { + return RubyFixnum.newFixnum(r, Short.toUnsignedLong((short) l)); + } + + private static IRubyObject ubyteToNum(Ruby r, long l) { + return RubyFixnum.newFixnum(r, Byte.toUnsignedLong((byte) l)); + } + + private static long num2FloatBits(IRubyObject d) { + return Float.floatToIntBits((float) RubyNumeric.num2dbl(d)); + } + + private static long num2DoubleBits(IRubyObject d) { + return Double.doubleToLongBits(RubyNumeric.num2dbl(d)); + } + } + + private static long swapAsShort(long l) { + short s = (short) l; + + return (s >>> 8) | (int) (s << 8); + } + + private static short swap(short s) { + return (short) ((s >>> 8) | (s << 8)); + } + + private static long swapAsInt(long l) { + int s = (int) l; + + return (s >>> 16) | (int) (s << 16); + } + + private static long swapAsLong(long l) { + return (l >>> 32) | (l << 32); + } + + private static long swapAsFloat(long l) { + return swapAsInt(l); + } + + private static long swapAsDouble(long l) { + return swapAsLong(l); + } + + private ByteBuffer base; private int size; private int flags; + private IRubyObject source; } diff --git a/core/src/main/java/org/jruby/runtime/Helpers.java b/core/src/main/java/org/jruby/runtime/Helpers.java index c836b06e930..1e99920cceb 100644 --- a/core/src/main/java/org/jruby/runtime/Helpers.java +++ b/core/src/main/java/org/jruby/runtime/Helpers.java @@ -2926,6 +2926,18 @@ public static IRubyObject[] arrayOf(IRubyObject first, IRubyObject... values) { return newValues; } + public static IRubyObject[] arrayOf(IRubyObject first) { + return new IRubyObject[] {first}; + } + + public static IRubyObject[] arrayOf(IRubyObject first, IRubyObject second) { + return new IRubyObject[] {first, second}; + } + + public static IRubyObject[] arrayOf(IRubyObject first, IRubyObject second, IRubyObject third) { + return new IRubyObject[] {first, second, third}; + } + public static T[] arrayOf(T[] values, T last, IntFunction allocator) { T[] newValues = allocator.apply(values.length + 1); newValues[values.length] = last; From ae776bf9a1b660ff01570bf8ab0142b431681e66 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 16:34:25 -0500 Subject: [PATCH 09/56] Add string sets/gets logic and fix errors --- .../src/main/java/org/jruby/RubyIOBuffer.java | 184 ++++++++++++------ core/src/main/java/org/jruby/RubyString.java | 4 + 2 files changed, 124 insertions(+), 64 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index d3ec4848f61..71b3ebd8486 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -4,6 +4,8 @@ import jnr.ffi.Platform; import jnr.ffi.Runtime; import jnr.ffi.Type; +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; import org.jruby.anno.JRubyConstant; import org.jruby.anno.JRubyMethod; import org.jruby.runtime.Arity; @@ -11,6 +13,7 @@ import org.jruby.runtime.Helpers; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.ByteList; import org.jruby.util.func.ObjectLongFunction; import java.math.BigInteger; @@ -96,12 +99,22 @@ public IRubyObject initialize(ThreadContext context, IRubyObject size) { } @JRubyMethod(name = "initialize") - public IRubyObject initialize(ThreadContext context, IRubyObject size, IRubyObject flags) { - return initialize(context, size.convertToInteger().getIntValue(), flags.convertToInteger().getIntValue(), context.nil); + public IRubyObject initialize(ThreadContext context, IRubyObject _size, IRubyObject flags) { + IRubyObject nil = context.nil; + + int size = _size.convertToInteger().getIntValue(); + + initialize(context, new byte[size], size, flags.convertToInteger().getIntValue(), nil); + + return nil; } public IRubyObject initialize(ThreadContext context, int size) { - return initialize(context, size, flagsForSize(size), context.nil); + IRubyObject nil = context.nil; + + initialize(context, new byte[size], size, flagsForSize(size), nil); + + return nil; } // MRI: io_buffer_initialize @@ -350,6 +363,7 @@ public IRubyObject slice(ThreadContext context, int offset, int length) { return newBuffer(context.runtime, slice, length, flags); } + // MRI: io_buffer_validate_range private void validateRange(ThreadContext context, int offset, int length) { if (offset + length > size) { throw context.runtime.newArgumentError("Specified offset+length exceeds data size!"); @@ -1041,43 +1055,106 @@ public IRubyObject copy(ThreadContext context, RubyIOBuffer source, int offset, return RubyFixnum.newFixnum(context.runtime, length); } + public IRubyObject copy(ThreadContext context, RubyString source, int offset, int length, int sourceOffset) { + if (sourceOffset > length) { + throw context.runtime.newArgumentError("The given source offset is bigger than the source itself!"); + } + + bufferCopy(context, offset, source.getByteList(), sourceOffset, source.size(), length); + + return RubyFixnum.newFixnum(context.runtime, length); + } + private void bufferCopy(ThreadContext context, int offset, ByteBuffer sourceBuffer, int sourceOffset, int sourceSize, int length) { ByteBuffer destBuffer = getBufferForWriting(context); sourceBuffer.position(sourceOffset); sourceBuffer.limit(sourceOffset + length); - destBuffer.put(sourceBuffer); + if (offset == 0) { + destBuffer.put(sourceBuffer); + } else { + destBuffer.position(offset); + destBuffer.put(sourceBuffer); + destBuffer.clear(); + } sourceBuffer.clear(); } + private void bufferCopy(ThreadContext context, int offset, ByteList sourceBuffer, int sourceOffset, int sourceSize, int length) { + ByteBuffer destBuffer = getBufferForWriting(context); + + if (offset == 0) { + destBuffer.put(sourceBuffer.getUnsafeBytes(), sourceBuffer.begin() + sourceOffset, length); + } else { + destBuffer.position(offset); + destBuffer.put(sourceBuffer.getUnsafeBytes(), sourceBuffer.begin() + sourceOffset, length); + destBuffer.clear(); + } + } + @JRubyMethod(name = "get_string") - public IRubyObject get_string(ThreadContext context, IRubyObject offset) { - return context.nil; + public IRubyObject get_string(ThreadContext context, IRubyObject _offset) { + int offset = RubyNumeric.num2int(_offset); + + return getString(context, offset, size, ASCIIEncoding.INSTANCE); } @JRubyMethod(name = "get_string") - public IRubyObject get_string(ThreadContext context, IRubyObject offset, IRubyObject length) { - return context.nil; + public IRubyObject get_string(ThreadContext context, IRubyObject _offset, IRubyObject _length) { + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + + return getString(context, offset, length, ASCIIEncoding.INSTANCE); } @JRubyMethod(name = "get_string") - public IRubyObject get_string(ThreadContext context, IRubyObject offset, IRubyObject length, IRubyObject encoding) { - return context.nil; + public IRubyObject get_string(ThreadContext context, IRubyObject _offset, IRubyObject _length, IRubyObject _encoding) { + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + Encoding encoding = context.runtime.getEncodingService().getEncodingFromObject(_encoding); + + return getString(context, offset, length, encoding); + } + + private IRubyObject getString(ThreadContext context, int offset, int length, Encoding encoding) { + ByteBuffer buffer = getBufferForReading(context); + + validateRange(context, offset, length); + + byte[] bytes = new byte[length]; + if (offset == 0) { + buffer.get(bytes, 0, length); + } else { + buffer.position(offset); + buffer.get(bytes, 0, length); + buffer.clear(); + } + + return RubyString.newString(context.runtime, bytes, 0, length, encoding); } @JRubyMethod(name = "set_string") - public IRubyObject set_string(ThreadContext context, IRubyObject string) { - return context.nil; + public IRubyObject set_string(ThreadContext context, IRubyObject _string) { + RubyString string = _string.convertToString(); + + return copy(context, string, 0, string.size(), 0); } @JRubyMethod(name = "set_string") - public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyObject offset) { - return context.nil; + public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyObject _offset) { + RubyString string = _string.convertToString(); + int offset = RubyNumeric.num2int(_offset); + + return copy(context, string, offset, string.size(), 0); } @JRubyMethod(name = "set_string") - public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyObject offset, IRubyObject length) { - return context.nil; + public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyObject _offset, IRubyObject _length) { + RubyString string = _string.convertToString(); + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + + return copy(context, string, offset, length, 0); } @JRubyMethod(name = "set_string", required = 1, optional = 3, checkArity = false) @@ -1098,8 +1175,13 @@ public IRubyObject set_string(ThreadContext context, IRubyObject[] args) { return context.nil; } - public IRubyObject set_string(ThreadContext context, IRubyObject string, IRubyObject offset, IRubyObject length, IRubyObject encoding) { - return context.nil; + public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyObject _offset, IRubyObject _length, IRubyObject _stringOffset) { + RubyString string = _string.convertToString(); + int offset = RubyNumeric.num2int(_offset); + int length = RubyNumeric.num2int(_length); + int stringOffset = RubyNumeric.num2int(_stringOffset); + + return copy(context, string, offset, length, stringOffset); } @JRubyMethod(name = "&") @@ -1189,58 +1271,32 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject fro } enum DataType { - U8(NativeType.UCHAR, BIG_ENDIAN, DataType::ubyteToNum, RubyNumeric::num2ulong, LongUnaryOperator.identity()), - S8(NativeType.SCHAR, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2long, LongUnaryOperator.identity()), - u16(NativeType.USHORT, LITTLE_ENDIAN, DataType::ushortToNum, RubyNumeric::num2ulong, DataType::swapAsShort), - U16(NativeType.USHORT, BIG_ENDIAN, DataType::ushortToNum, RubyNumeric::num2ulong, DataType::swapAsShort), - s16(NativeType.SSHORT, LITTLE_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsShort), - S16(NativeType.SSHORT, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsShort), - u32(NativeType.UINT, LITTLE_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsInt), - U32(NativeType.UINT, BIG_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsInt), - s32(NativeType.SINT, LITTLE_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsInt), - S32(NativeType.SINT, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsInt), - u64(NativeType.ULONG, LITTLE_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsLong), - U64(NativeType.ULONG, BIG_ENDIAN, DataType::uintToNum, RubyNumeric::num2ulong, DataType::swapAsLong), - s64(NativeType.SLONG, LITTLE_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsLong), - S64(NativeType.SLONG, BIG_ENDIAN, RubyFixnum::newFixnum, RubyNumeric::num2ulong, DataType::swapAsLong), - f32(NativeType.FLOAT,LITTLE_ENDIAN, RubyFloat::newFloat, DataType::num2FloatBits, DataType::swapAsInt), - F32(NativeType.FLOAT, BIG_ENDIAN, RubyFloat::newFloat, DataType::num2FloatBits, DataType::swapAsInt), - f64(NativeType.DOUBLE, LITTLE_ENDIAN, RubyFloat::newFloat, DataType::num2DoubleBits, DataType::swapAsLong), - F64(NativeType.DOUBLE, BIG_ENDIAN, RubyFloat::newFloat, DataType::num2DoubleBits, DataType::swapAsLong); - - DataType(NativeType type, int endian, ObjectLongFunction toNum, ToLongFunction fromNum, LongUnaryOperator swap, Function) { + U8(NativeType.UCHAR, BIG_ENDIAN), + S8(NativeType.SCHAR, BIG_ENDIAN), + u16(NativeType.USHORT, LITTLE_ENDIAN), + U16(NativeType.USHORT, BIG_ENDIAN), + s16(NativeType.SSHORT, LITTLE_ENDIAN), + S16(NativeType.SSHORT, BIG_ENDIAN), + u32(NativeType.UINT, LITTLE_ENDIAN), + U32(NativeType.UINT, BIG_ENDIAN), + s32(NativeType.SINT, LITTLE_ENDIAN), + S32(NativeType.SINT, BIG_ENDIAN), + u64(NativeType.ULONG, LITTLE_ENDIAN), + U64(NativeType.ULONG, BIG_ENDIAN), + s64(NativeType.SLONG, LITTLE_ENDIAN), + S64(NativeType.SLONG, BIG_ENDIAN), + f32(NativeType.FLOAT,LITTLE_ENDIAN), + F32(NativeType.FLOAT, BIG_ENDIAN), + f64(NativeType.DOUBLE, LITTLE_ENDIAN), + F64(NativeType.DOUBLE, BIG_ENDIAN); + + DataType(NativeType type, int endian) { this.type = FFI_RUNTIME.findType(type); this.endian = endian; - this.toNum = toNum; - this.fromNum = fromNum; - this.swap = swap; } private final Type type; private final int endian; - private final ObjectLongFunction toNum; - private final ToLongFunction fromNum; - private final LongUnaryOperator swap; - - private static IRubyObject uintToNum(Ruby r, long l) { - return RubyFixnum.newFixnum(r, Integer.toUnsignedLong((int) l)); - } - - private static IRubyObject ushortToNum(Ruby r, long l) { - return RubyFixnum.newFixnum(r, Short.toUnsignedLong((short) l)); - } - - private static IRubyObject ubyteToNum(Ruby r, long l) { - return RubyFixnum.newFixnum(r, Byte.toUnsignedLong((byte) l)); - } - - private static long num2FloatBits(IRubyObject d) { - return Float.floatToIntBits((float) RubyNumeric.num2dbl(d)); - } - - private static long num2DoubleBits(IRubyObject d) { - return Double.doubleToLongBits(RubyNumeric.num2dbl(d)); - } } private static long swapAsShort(long l) { diff --git a/core/src/main/java/org/jruby/RubyString.java b/core/src/main/java/org/jruby/RubyString.java index 02a954b31b8..e762a6eafd4 100644 --- a/core/src/main/java/org/jruby/RubyString.java +++ b/core/src/main/java/org/jruby/RubyString.java @@ -513,6 +513,10 @@ public static RubyString newString(Ruby runtime, byte[] bytes, int start, int le return new RubyString(runtime, runtime.getString(), new ByteList(copy, encoding, false)); } + public static RubyString newStringNoCopy(Ruby runtime, byte[] bytes, int start, int length, Encoding encoding) { + return new RubyString(runtime, runtime.getString(), new ByteList(bytes, start, length, encoding, false)); + } + public static RubyString newString(Ruby runtime, ByteList bytes) { return new RubyString(runtime, runtime.getString(), bytes); } From ae5a6faf1aa242f8f539deb52971da2da61532bb Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 16:59:39 -0500 Subject: [PATCH 10/56] Bit masking logic --- .../src/main/java/org/jruby/RubyIOBuffer.java | 90 +++++++++++++++---- 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 71b3ebd8486..6016e7101db 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -68,7 +68,11 @@ public static RubyClass createIOBufferClass(Ruby runtime) { public static final int NETWORK_ENDIAN = BIG_ENDIAN; public static RubyIOBuffer newBuffer(Ruby runtime, ByteBuffer base, int size, int flags) { - return new RubyIOBuffer(runtime, runtime.getIO(), base, size, flags); + return new RubyIOBuffer(runtime, runtime.getIOBuffer(), base, size, flags); + } + + public static RubyIOBuffer newBuffer(Ruby runtime, int size, int flags) { + return new RubyIOBuffer(runtime, runtime.getIOBuffer(), newBufferBase(runtime, size, flags), size, flags); } public RubyIOBuffer(Ruby runtime, RubyClass metaClass) { @@ -125,17 +129,7 @@ public void initialize(ThreadContext context, byte[] baseBytes, int size, int fl // If we are provided a pointer, we use it. base = ByteBuffer.wrap(baseBytes); } else if (size != 0) { - // If we are provided a non-zero size, we allocate it: - if ((flags & INTERNAL) == INTERNAL) { - base = ByteBuffer.allocate(size); - } else if ((flags & MAPPED) == MAPPED) { - // no support for SHARED, PRIVATE yet - base = ByteBuffer.allocateDirect(size); - } - - if (base == null) { - throw context.runtime.newBufferAllocationError("Could not allocate buffer!"); - } + base = newBufferBase(context.runtime, size, flags, base); } else { // Otherwise we don't do anything. return; @@ -147,6 +141,22 @@ public void initialize(ThreadContext context, byte[] baseBytes, int size, int fl this.source = source; } + private static ByteBuffer newBufferBase(Ruby runtime, int size, int flags) { + ByteBuffer base; + + // If we are provided a non-zero size, we allocate it: + if ((flags & INTERNAL) == INTERNAL) { + base = ByteBuffer.allocate(size); + } else if ((flags & MAPPED) == MAPPED) { + // no support for SHARED, PRIVATE yet + base = ByteBuffer.allocateDirect(size); + } else { + throw runtime.newBufferAllocationError("Could not allocate buffer!"); + } + + return base; + } + // MRI: io_flags_for_size private static int flagsForSize(int size) { if (size >= PAGE_SIZE) { @@ -1185,23 +1195,65 @@ public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyO } @JRubyMethod(name = "&") - public IRubyObject op_and(ThreadContext context, IRubyObject mask) { - return context.nil; + public IRubyObject op_and(ThreadContext context, IRubyObject _mask) { + ByteBuffer buffer = getBufferForReading(context); + + int mask = _mask.convertToInteger().getIntValue(); + + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); + ByteBuffer output = outputBuffer.base; + + for (int i = 0; i < buffer.capacity(); i++) { + output.put(i, (byte) (buffer.get(i) & mask)); + } + + return outputBuffer; } @JRubyMethod(name = "|") - public IRubyObject op_or(ThreadContext context, IRubyObject mask) { - return context.nil; + public IRubyObject op_or(ThreadContext context, IRubyObject _mask) { + ByteBuffer buffer = getBufferForReading(context); + + int mask = _mask.convertToInteger().getIntValue(); + + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); + ByteBuffer output = outputBuffer.base; + + for (int i = 0; i < buffer.capacity(); i++) { + output.put(i, (byte) (buffer.get(i) | mask)); + } + + return outputBuffer; } @JRubyMethod(name = "^") - public IRubyObject op_xor(ThreadContext context, IRubyObject mask) { - return context.nil; + public IRubyObject op_xor(ThreadContext context, IRubyObject _mask) { + ByteBuffer buffer = getBufferForReading(context); + + int mask = _mask.convertToInteger().getIntValue(); + + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); + ByteBuffer output = outputBuffer.base; + + for (int i = 0; i < buffer.capacity(); i++) { + output.put(i, (byte) (buffer.get(i) ^ mask)); + } + + return outputBuffer; } @JRubyMethod(name = "~") public IRubyObject op_not(ThreadContext context) { - return context.nil; + ByteBuffer buffer = getBufferForReading(context); + + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); + ByteBuffer output = outputBuffer.base; + + for (int i = 0; i < buffer.capacity(); i++) { + output.put(i, (byte) ~buffer.get(i)); + } + + return outputBuffer; } @JRubyMethod(name = "read") From 7b175dc3dffe1a911acc2ef53b6a1b4730de4486 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 17:53:15 -0500 Subject: [PATCH 11/56] IO::Buffer read and write logic --- .../main/java/org/jruby/FiberScheduler.java | 48 +++++++ .../src/main/java/org/jruby/RubyIOBuffer.java | 88 ++++++++++++- core/src/main/java/org/jruby/RubyThread.java | 42 +++++-- .../main/java/org/jruby/util/io/OpenFile.java | 119 +++++++++++++++--- .../java/org/jruby/util/io/PosixShim.java | 109 ++++++++++------ 5 files changed, 333 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index 3d2ad2f63aa..a1e239fa878 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -5,6 +5,8 @@ import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.io.OpenFile; +import java.nio.ByteBuffer; + public class FiberScheduler { // MRI: rb_fiber_scheduler_kernel_sleep public static IRubyObject kernelSleep(ThreadContext context, IRubyObject scheduler, IRubyObject timeout) { @@ -61,23 +63,50 @@ public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, I return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, context.runtime.newFixnum(length)); } + public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger length, RubyInteger offset) { + return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, length, offset); + } + // MRI: rb_fiber_scheduler_io_pread public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); } + public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, IRubyObject length, IRubyObject offset) { + return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, length, offset); + } + // MRI: rb_fiber_scheduler_io_write public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length) { return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, context.runtime.newFixnum(length)); } + public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger length, RubyInteger offset) { + return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, length, offset); + } + // MRI: rb_fiber_scheduler_io_pwrite public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); } + public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, IRubyObject length, IRubyObject offset) { + return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, length, offset); + } + // MRI: rb_fiber_scheduler_io_read_memory public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, byte[] base, int size, int length) { + RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, ByteBuffer.wrap(base), size, RubyIOBuffer.LOCKED); + + IRubyObject result = ioRead(context, scheduler, io, buffer, length); + + buffer.unlock(context); + buffer.free(context); + + return result; + } + + public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int size, int length) { RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED); IRubyObject result = ioRead(context, scheduler, io, buffer, length); @@ -90,6 +119,17 @@ public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject schedu // MRI: rb_fiber_scheduler_io_write_memory public static IRubyObject ioWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, byte[] base, int size, int length) { + RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, ByteBuffer.wrap(base), size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); + + IRubyObject result = ioWrite(context, scheduler, io, buffer, length); + + buffer.unlock(context); + buffer.free(context); + + return result; + } + + public static IRubyObject ioWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int size, int length) { RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); IRubyObject result = ioWrite(context, scheduler, io, buffer, length); @@ -154,4 +194,12 @@ public static int resultApply(ThreadContext context, IRubyObject result) { return RubyNumeric.num2int(result); } } + + public static IRubyObject result(Ruby runtime, int result, int error) { + if (result == -1) { + return RubyFixnum.newFixnum(runtime, error); + } else { + return RubyFixnum.newFixnum(runtime, error); + } + } } diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 6016e7101db..3e30fc517ef 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -15,7 +15,10 @@ import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.ByteList; import org.jruby.util.func.ObjectLongFunction; +import org.jruby.util.io.OpenFile; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -129,7 +132,7 @@ public void initialize(ThreadContext context, byte[] baseBytes, int size, int fl // If we are provided a pointer, we use it. base = ByteBuffer.wrap(baseBytes); } else if (size != 0) { - base = newBufferBase(context.runtime, size, flags, base); + base = newBufferBase(context.runtime, size, flags); } else { // Otherwise we don't do anything. return; @@ -1258,12 +1261,48 @@ public IRubyObject op_not(ThreadContext context) { @JRubyMethod(name = "read") public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length) { - return context.nil; + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger lengthInteger = length.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioRead(context, scheduler, io, this, lengthInteger, RubyFixnum.zero(context.runtime)); + + if (result != UNDEF) { + return result; + } + } + + return read(context, io, lengthInteger.getIntValue(), 0); } @JRubyMethod(name = "read") public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length, IRubyObject offset) { - return context.nil; + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger lengthInteger = length.convertToInteger(); + RubyInteger offsetInteger = offset.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioRead(context, scheduler, io, this, lengthInteger, offsetInteger); + + if (result != UNDEF) { + return result; + } + } + + return read(context, io, lengthInteger.getIntValue(), offsetInteger.getIntValue()); + } + + public IRubyObject read(ThreadContext context, IRubyObject io, int length, int offset) { + validateRange(context, offset, length); + + ByteBuffer buffer = getBufferForWriting(context); + + return readInternal(context, RubyIO.convertToIO(context, io), buffer, offset, length); + } + + private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBuffer base, int offset, int size) { + int result = OpenFile.readInternal(context, io.openFile, io.openFile.fd(), base, offset, size); + return FiberScheduler.result(context.runtime, result, io.openFile.posix.getErrno().value()); } @JRubyMethod(name = "pread") @@ -1286,17 +1325,54 @@ public IRubyObject pread(ThreadContext context, IRubyObject[] args) { } public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length, IRubyObject offset) { - return context.nil; + throw context.runtime.newNotImplementedError("pread"); } @JRubyMethod(name = "write") public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject length) { - return context.nil; + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger lengthInteger = length.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioWrite(context, scheduler, io, this, lengthInteger, RubyFixnum.zero(context.runtime)); + + if (result != UNDEF) { + return result; + } + } + + return write(context, io, lengthInteger.getIntValue(), 0); } @JRubyMethod(name = "write") public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject length, IRubyObject offset) { - return context.nil; + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger lengthInteger = length.convertToInteger(); + RubyInteger offsetInteger = offset.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioWrite(context, scheduler, io, this, lengthInteger, offsetInteger); + + if (result != UNDEF) { + return result; + } + } + + return write(context, io, lengthInteger.getIntValue(), offsetInteger.getIntValue()); + } + + public IRubyObject write(ThreadContext context, IRubyObject io, int length, int offset) { + validateRange(context, offset, length); + + + ByteBuffer buffer = getBufferForWriting(context); + + return writeInternal(context, RubyIO.convertToIO(context, io), buffer, offset, length); + } + + private static IRubyObject writeInternal(ThreadContext context, RubyIO io, ByteBuffer base, int offset, int size) { + int result = OpenFile.writeInternal(context, io.openFile, base, offset, size); + return FiberScheduler.result(context.runtime, result, io.openFile.posix.getErrno().value()); } @JRubyMethod(name = "pwrite") diff --git a/core/src/main/java/org/jruby/RubyThread.java b/core/src/main/java/org/jruby/RubyThread.java index 49b3de386cd..63b6443a391 100644 --- a/core/src/main/java/org/jruby/RubyThread.java +++ b/core/src/main/java/org/jruby/RubyThread.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; import java.nio.channels.Channel; import java.nio.channels.SelectableChannel; import java.nio.channels.SelectionKey; @@ -1789,25 +1790,48 @@ public int executeReadWrite( ReadWrite task) throws InterruptedException { Status oldStatus = STATUS.get(this); try { - this.unblockArg = data; - this.unblockFunc = task; + preReadWrite(context, data, task); - // check for interrupt before going into blocking call - blockingThreadPoll(context); + return task.run(context, data, bytes, start, length); + } finally { + postReadWrite(context, oldStatus); + } + } - STATUS.set(this, Status.SLEEP); + public int executeReadWrite( + ThreadContext context, + Data data, ByteBuffer bytes, int start, int length, + ReadWrite task) throws InterruptedException { + Status oldStatus = STATUS.get(this); + try { + preReadWrite(context, data, task); return task.run(context, data, bytes, start, length); } finally { - STATUS.set(this, oldStatus); - this.unblockFunc = null; - this.unblockArg = null; - pollThreadEvents(context); + postReadWrite(context, oldStatus); } } + private void postReadWrite(ThreadContext context, Status oldStatus) { + STATUS.set(this, oldStatus); + this.unblockFunc = null; + this.unblockArg = null; + pollThreadEvents(context); + } + + private void preReadWrite(ThreadContext context, Data data, ReadWrite task) { + this.unblockArg = data; + this.unblockFunc = task; + + // check for interrupt before going into blocking call + blockingThreadPoll(context); + + STATUS.set(this, Status.SLEEP); + } + public interface ReadWrite extends Unblocker { public int run(ThreadContext context, Data data, byte[] bytes, int start, int length) throws InterruptedException; + public int run(ThreadContext context, Data data, ByteBuffer bytes, int start, int length) throws InterruptedException; public void wakeup(RubyThread thread, Data data); } diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index 9c6f628971b..da2462e5e22 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -2,6 +2,7 @@ import java.io.Closeable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.channels.Channel; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; @@ -1334,6 +1335,25 @@ && waitReadable(context, fd)) { final static RubyThread.ReadWrite READ_TASK = new RubyThread.ReadWrite() { @Override public int run(ThreadContext context, OpenFile fptr, byte[] buffer, int start, int length) throws InterruptedException { + ChannelFD fd = preRead(context, fptr); + try { + return fptr.posix.read(fd, buffer, start, length, fptr.nonblock); + } finally { + fptr.lock(); + } + } + + @Override + public int run(ThreadContext context, OpenFile fptr, ByteBuffer buffer, int start, int length) throws InterruptedException { + ChannelFD fd = preRead(context, fptr); + try { + return fptr.posix.read(fd, buffer, start, length, fptr.nonblock); + } finally { + fptr.lock(); + } + } + + private ChannelFD preRead(ThreadContext context, OpenFile fptr) { ChannelFD fd = fptr.fd; if (fd == null) { @@ -1344,11 +1364,7 @@ public int run(ThreadContext context, OpenFile fptr, byte[] buffer, int start, i assert fptr.lockedByMe(); fptr.unlock(); - try { - return fptr.posix.read(fd, buffer, start, length, fptr.nonblock); - } finally { - fptr.lock(); - } + return fd; } @Override @@ -1360,6 +1376,25 @@ public void wakeup(RubyThread thread, OpenFile data) { final static RubyThread.ReadWrite WRITE_TASK = new RubyThread.ReadWrite() { @Override public int run(ThreadContext context, OpenFile fptr, byte[] bytes, int start, int length) throws InterruptedException { + ChannelFD fd = preWrite(context, fptr); + try { + return fptr.posix.write(fd, bytes, start, length, fptr.nonblock); + } finally { + fptr.lock(); + } + } + + @Override + public int run(ThreadContext context, OpenFile fptr, ByteBuffer bytes, int start, int length) throws InterruptedException { + ChannelFD fd = preWrite(context, fptr); + try { + return fptr.posix.write(fd, bytes, start, length, fptr.nonblock); + } finally { + fptr.lock(); + } + } + + private ChannelFD preWrite(ThreadContext context, OpenFile fptr) { ChannelFD fd = fptr.fd; if (fd == null) { @@ -1370,11 +1405,7 @@ public int run(ThreadContext context, OpenFile fptr, byte[] bytes, int start, in assert fptr.lockedByMe(); fptr.unlock(); - try { - return fptr.posix.write(fd, bytes, start, length, fptr.nonblock); - } finally { - fptr.lock(); - } + return fd; } @Override @@ -1384,7 +1415,7 @@ public void wakeup(RubyThread thread, OpenFile data) { } }; - // rb_read_internal, rb_io_read_memory + // rb_read_internal, rb_io_read_memory, rb_io_buffer_read_internal public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, byte[] bufBytes, int buf, int count) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); if (!scheduler.isNil()) { @@ -1407,6 +1438,48 @@ simple read(2) because EINTR does not damage the descriptor. working with any native descriptor. */ + preRead(context, fptr, fd); + + try { + return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, READ_TASK); + } catch (InterruptedException ie) { + throw context.runtime.newConcurrencyError("IO operation interrupted"); + } + } + + // rb_io_buffer_read_internal + public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer bufBytes, int buf, int count) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + + if (result != RubyBasicObject.UNDEF) { + FiberScheduler.resultApply(context, result); + } + } + // if we can do selection and this is not a non-blocking call, do selection + + /* + NOTE CON: We only do this selection because on the JDK, blocking calls to NIO channels can't be + interrupted, and we need to be able to interrupt blocking reads. In MRI, this logic is always just a + simple read(2) because EINTR does not damage the descriptor. + + Doing selects here on ENXIO native channels caused FIFOs to block for read all the time, even when no + writers are connected. This may or may not be correct behavior for selects against FIFOs, but in any + case since MRI does not do a select here at all I believe correct logic is to skip the select when + working with any native descriptor. + */ + + preRead(context, fptr, fd); + + try { + return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, READ_TASK); + } catch (InterruptedException ie) { + throw context.runtime.newConcurrencyError("IO operation interrupted"); + } + } + + private static void preRead(ThreadContext context, OpenFile fptr, ChannelFD fd) { if (fd == null) { // stream was closed on its way in, raise appropriate error throw context.runtime.newErrnoEBADFError(); @@ -1422,12 +1495,6 @@ simple read(2) because EINTR does not damage the descriptor. } finally { fptr.lock(); } - - try { - return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, READ_TASK); - } catch (InterruptedException ie) { - throw context.runtime.newConcurrencyError("IO operation interrupted"); - } } /** @@ -2351,6 +2418,24 @@ public static int writeInternal(ThreadContext context, OpenFile fptr, byte[] buf } } + // rb_io_buffer_write_internal + public static int writeInternal(ThreadContext context, OpenFile fptr, ByteBuffer bufBytes, int buf, int count) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + + if (result != RubyBasicObject.UNDEF) { + FiberScheduler.resultApply(context, result); + } + } + + try { + return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, WRITE_TASK); + } catch (InterruptedException ie) { + throw context.runtime.newConcurrencyError("IO operation interrupted"); + } + } + // rb_write_internal2 (no GVL version...we just don't use executeTask as above. int writeInternal2(ChannelFD fd, byte[] bufBytes, int buf, int count) { return posix.write(fd, bufBytes, buf, count, nonblock); diff --git a/core/src/main/java/org/jruby/util/io/PosixShim.java b/core/src/main/java/org/jruby/util/io/PosixShim.java index ae0c72e6ee4..ed3c36cba3a 100644 --- a/core/src/main/java/org/jruby/util/io/PosixShim.java +++ b/core/src/main/java/org/jruby/util/io/PosixShim.java @@ -89,10 +89,15 @@ public long lseek(ChannelFD fd, long offset, int type) { } public int write(ChannelFD fd, byte[] bytes, int offset, int length, boolean nonblock) { - clear(); - // FIXME: don't allocate every time ByteBuffer tmp = ByteBuffer.wrap(bytes, offset, length); + + return write(fd, tmp, offset, length, nonblock); + } + + public int write(ChannelFD fd, ByteBuffer buffer, int offset, int length, boolean nonblock) { + clear(); + try { if (nonblock) { // TODO: figure out what nonblocking writes against atypical streams (files?) actually do @@ -104,7 +109,7 @@ public int write(ChannelFD fd, byte[] bytes, int offset, int length, boolean non setErrno(Errno.EACCES); return -1; } - int written = fd.chWrite.write(tmp); + int written = fd.chWrite.write(buffer); if (written == 0 && length > 0) { // if it's a nonblocking write against a file and we've hit EOF, do EAGAIN @@ -129,55 +134,77 @@ public int read(ChannelFD fd, byte[] target, int offset, int length, boolean non clear(); try { - if (nonblock) { - // need to ensure channels that don't support nonblocking IO at least - // appear to be nonblocking - if (fd.chSelect != null) { - // ok...we should have set it nonblocking already in setNonblock - } else { - if (fd.chFile != null) { - long position = fd.chFile.position(); - long size = fd.chFile.size(); - if (position != -1 && size != -1 && position < size) { - // there should be bytes available...proceed - } else { - setErrno(Errno.EAGAIN); - return -1; - } - } else if (fd.chNative != null && fd.isNativeFile) { - // it's a native file, so we don't do selection or nonblock - } else { - setErrno(Errno.EAGAIN); - return -1; - } - } - } + if (checkForBlocking(fd, nonblock)) return -1; // FIXME: inefficient to recreate ByteBuffer every time ByteBuffer buffer = ByteBuffer.wrap(target, offset, length); - int read = fd.chRead.read(buffer); + return performRead(fd, buffer, nonblock); + } catch (IOException ioe) { + setErrno(Helpers.errnoFromException(ioe)); + return -1; + } + } - if (nonblock) { - if (read == JAVA_EOF) { - read = NATIVE_EOF; // still treat EOF as EOF - } else if (read == 0) { - setErrno(Errno.EAGAIN); - return -1; - } else { - return read; - } - } else { - // NIO channels will always raise for errors, so -1 only means EOF. - if (read == JAVA_EOF) read = NATIVE_EOF; - } + public int read(ChannelFD fd, ByteBuffer buffer, int offset, int length, boolean nonblock) { + clear(); + + try { + if (checkForBlocking(fd, nonblock)) return -1; - return read; + return performRead(fd, buffer, nonblock); } catch (IOException ioe) { setErrno(Helpers.errnoFromException(ioe)); return -1; } } + private int performRead(ChannelFD fd, ByteBuffer buffer, boolean nonblock) throws IOException { + int read = fd.chRead.read(buffer); + + if (nonblock) { + if (read == JAVA_EOF) { + read = NATIVE_EOF; // still treat EOF as EOF + } else if (read == 0) { + setErrno(Errno.EAGAIN); + return -1; + } else { + return read; + } + } else { + // NIO channels will always raise for errors, so -1 only means EOF. + if (read == JAVA_EOF) read = NATIVE_EOF; + } + + return read; + } + + private boolean checkForBlocking(ChannelFD fd, boolean nonblock) throws IOException { + if (nonblock) { + // need to ensure channels that don't support nonblocking IO at least + // appear to be nonblocking + if (fd.chSelect != null) { + // ok...we should have set it nonblocking already in setNonblock + } else { + if (fd.chFile != null) { + long position = fd.chFile.position(); + long size = fd.chFile.size(); + if (position != -1 && size != -1 && position < size) { + // there should be bytes available...proceed + } else { + setErrno(Errno.EAGAIN); + return true; + } + } else if (fd.chNative != null && fd.isNativeFile) { + // it's a native file, so we don't do selection or nonblock + } else { + setErrno(Errno.EAGAIN); + return true; + } + } + } + return false; + } + // rb_thread_flock public int flock(ChannelFD fd, int lockMode) { // TODO: null channel always succeeds for all locking operations From fee8930b6f2698dc4c4aa68b1cc147cc49409fcd Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 18:36:12 -0500 Subject: [PATCH 12/56] Update IO::Buffer test from 3.3 HEAD --- test/mri/ruby/test_io_buffer.rb | 178 +++++++++++++++++++++++++++----- 1 file changed, 151 insertions(+), 27 deletions(-) diff --git a/test/mri/ruby/test_io_buffer.rb b/test/mri/ruby/test_io_buffer.rb index 7e3b467ed59..d9fa22fd2a4 100644 --- a/test/mri/ruby/test_io_buffer.rb +++ b/test/mri/ruby/test_io_buffer.rb @@ -88,30 +88,34 @@ def test_file_mapped_invalid def test_string_mapped string = "Hello World" buffer = IO::Buffer.for(string) - refute buffer.readonly? - - # Cannot modify string as it's locked by the buffer: - assert_raise RuntimeError do - string[0] = "h" - end - - buffer.set_value(:U8, 0, "h".ord) - - # Buffer releases it's ownership of the string: - buffer.free - - assert_equal "hello World", string - string[0] = "H" - assert_equal "Hello World", string + assert buffer.readonly? end def test_string_mapped_frozen string = "Hello World".freeze buffer = IO::Buffer.for(string) - assert buffer.readonly? end + def test_string_mapped_mutable + string = "Hello World" + IO::Buffer.for(string) do |buffer| + refute buffer.readonly? + + # Cannot modify string as it's locked by the buffer: + assert_raise RuntimeError do + string[0] = "h" + end + + buffer.set_value(:U8, 0, "h".ord) + + # Buffer releases it's ownership of the string: + buffer.free + + assert_equal "hello World", string + end + end + def test_non_string not_string = Object.new @@ -165,16 +169,26 @@ def test_slice assert_equal("Hello World", buffer.get_string(8, 11)) end - def test_slice_bounds + def test_slice_arguments + buffer = IO::Buffer.for("Hello World") + + slice = buffer.slice + assert_equal "Hello World", slice.get_string + + slice = buffer.slice(2) + assert_equal("llo World", slice.get_string) + end + + def test_slice_bounds_error buffer = IO::Buffer.new(128) assert_raise ArgumentError do buffer.slice(128, 10) end - # assert_raise RuntimeError do - # pp buffer.slice(-10, 10) - # end + assert_raise ArgumentError do + buffer.slice(-10, 10) + end end def test_locked @@ -231,17 +245,59 @@ def test_get_string :F64 => [-1.0, 0.0, 0.5, 1.0, 128.0], } - def test_get_set_primitives + def test_get_set_value buffer = IO::Buffer.new(128) - RANGES.each do |type, values| + RANGES.each do |data_type, values| values.each do |value| - buffer.set_value(type, 0, value) - assert_equal value, buffer.get_value(type, 0), "Converting #{value} as #{type}." + buffer.set_value(data_type, 0, value) + assert_equal value, buffer.get_value(data_type, 0), "Converting #{value} as #{data_type}." end end end + def test_get_set_values + buffer = IO::Buffer.new(128) + + RANGES.each do |data_type, values| + format = [data_type] * values.size + + buffer.set_values(format, 0, values) + assert_equal values, buffer.get_values(format, 0), "Converting #{values} as #{format}." + end + end + + def test_values + buffer = IO::Buffer.new(128) + + RANGES.each do |data_type, values| + format = [data_type] * values.size + + buffer.set_values(format, 0, values) + assert_equal values, buffer.values(data_type, 0, values.size), "Reading #{values} as #{format}." + end + end + + def test_each + buffer = IO::Buffer.new(128) + + RANGES.each do |data_type, values| + format = [data_type] * values.size + data_type_size = IO::Buffer.size_of(data_type) + values_with_offsets = values.map.with_index{|value, index| [index * data_type_size, value]} + + buffer.set_values(format, 0, values) + assert_equal values_with_offsets, buffer.each(data_type, 0, values.size).to_a, "Reading #{values} as #{data_type}." + end + end + + def test_each_byte + string = "The quick brown fox jumped over the lazy dog." + buffer = IO::Buffer.for(string) + + assert_equal string.bytes, buffer.each_byte.to_a + end + def test_clear buffer = IO::Buffer.new(16) buffer.set_string("Hello World!") @@ -274,6 +330,10 @@ def test_invalidation end def test_read + # This is currently a bug in IO:Buffer [#19084] which affects extended + # strings. On 32 bit machines, the example below becomes extended, so + # we omit this test until the bug is fixed. + omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 io = Tempfile.new io.write("Hello World") io.seek(0) @@ -283,7 +343,7 @@ def test_read assert_equal "Hello", buffer.get_string(0, 5) ensure - io.close! + io.close! if io end def test_write @@ -305,7 +365,7 @@ def test_pread io.seek(0) buffer = IO::Buffer.new(128) - buffer.pread(io, 5, 6) + buffer.pread(io, 6, 5) assert_equal "World", buffer.get_string(0, 5) assert_equal 0, io.tell @@ -313,12 +373,41 @@ def test_pread io.close! end + def test_pread_offset + io = Tempfile.new + io.write("Hello World") + io.seek(0) + + buffer = IO::Buffer.new(128) + buffer.pread(io, 6, 5, 6) + + assert_equal "World", buffer.get_string(6, 5) + assert_equal 0, io.tell + ensure + io.close! + end + def test_pwrite io = Tempfile.new buffer = IO::Buffer.new(128) buffer.set_string("World") - buffer.pwrite(io, 5, 6) + buffer.pwrite(io, 6, 5) + + assert_equal 0, io.tell + + io.seek(6) + assert_equal "World", io.read(5) + ensure + io.close! + end + + def test_pwrite_offset + io = Tempfile.new + + buffer = IO::Buffer.new(128) + buffer.set_string("Hello World") + buffer.pwrite(io, 6, 5, 6) assert_equal 0, io.tell @@ -327,4 +416,39 @@ def test_pwrite ensure io.close! end + + def test_operators + source = IO::Buffer.for("1234123412") + mask = IO::Buffer.for("133\x00") + + assert_equal IO::Buffer.for("123\x00123\x0012"), (source & mask) + assert_equal IO::Buffer.for("1334133413"), (source | mask) + assert_equal IO::Buffer.for("\x00\x01\x004\x00\x01\x004\x00\x01"), (source ^ mask) + assert_equal IO::Buffer.for("\xce\xcd\xcc\xcb\xce\xcd\xcc\xcb\xce\xcd"), ~source + end + + def test_inplace_operators + source = IO::Buffer.for("1234123412") + mask = IO::Buffer.for("133\x00") + + assert_equal IO::Buffer.for("123\x00123\x0012"), source.dup.and!(mask) + assert_equal IO::Buffer.for("1334133413"), source.dup.or!(mask) + assert_equal IO::Buffer.for("\x00\x01\x004\x00\x01\x004\x00\x01"), source.dup.xor!(mask) + assert_equal IO::Buffer.for("\xce\xcd\xcc\xcb\xce\xcd\xcc\xcb\xce\xcd"), source.dup.not! + end + + def test_shared + message = "Hello World" + buffer = IO::Buffer.new(64, IO::Buffer::MAPPED | IO::Buffer::SHARED) + + pid = fork do + buffer.set_string(message) + end + + Process.wait(pid) + string = buffer.get_string(0, message.bytesize) + assert_equal message, string + rescue NotImplementedError + omit "Fork/shared memory is not supported." + end end From f3d903a893d72a3196bd5eea3d989991d19b9ef2 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 18:36:29 -0500 Subject: [PATCH 13/56] Clean up imports --- core/src/main/java/org/jruby/RubyIOBuffer.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 3e30fc517ef..aea9479ad50 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -14,17 +14,12 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.ByteList; -import org.jruby.util.func.ObjectLongFunction; import org.jruby.util.io.OpenFile; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -import java.util.function.LongUnaryOperator; -import java.util.function.ToLongFunction; import static org.jruby.RubyBoolean.newBoolean; From 1c23b8a93b0a44fe7dc9cdef91e2f7082ac1155c Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 29 Sep 2023 18:36:53 -0500 Subject: [PATCH 14/56] Start trying to pass test_io_buffer --- .../main/java/org/jruby/FiberScheduler.java | 7 +- core/src/main/java/org/jruby/RubyIO.java | 3 +- .../src/main/java/org/jruby/RubyIOBuffer.java | 75 ++++++++++++++++--- 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index a1e239fa878..4b5b29aae1f 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -1,5 +1,6 @@ package org.jruby; +import jnr.constants.platform.Errno; import org.jruby.runtime.Helpers; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; @@ -195,11 +196,11 @@ public static int resultApply(ThreadContext context, IRubyObject result) { } } - public static IRubyObject result(Ruby runtime, int result, int error) { + public static IRubyObject result(Ruby runtime, int result, Errno error) { if (result == -1) { - return RubyFixnum.newFixnum(runtime, error); + return RubyFixnum.newFixnum(runtime, error.value()); } else { - return RubyFixnum.newFixnum(runtime, error); + return RubyFixnum.newFixnum(runtime, result); } } } diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java index 55ca0416a9b..9556315d3dc 100644 --- a/core/src/main/java/org/jruby/RubyIO.java +++ b/core/src/main/java/org/jruby/RubyIO.java @@ -3137,7 +3137,8 @@ IRubyObject getPartial(ThreadContext context, IRubyObject[] args, boolean nonblo fptr = getOpenFileChecked(); - final boolean locked = fptr.lock(); int n; + final boolean locked = fptr.lock(); + int n; try { fptr.checkByteReadable(context); diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index aea9479ad50..e5f3f0ab51a 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -85,9 +85,28 @@ public RubyIOBuffer(Ruby runtime, RubyClass metaClass, ByteBuffer base, int size this.flags = flags; } - @JRubyMethod(name = "for") - public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyObject string) { - return context.nil; + @JRubyMethod(name = "for", meta = true) + public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyObject _string, Block block) { + RubyString string = _string.convertToString(); + + // If the string is frozen, both code paths are okay. + // If the string is not frozen, if a block is not given, it must be frozen. + boolean isGiven = block.isGiven(); + if (!isGiven) { + // This internally returns the source string if it's already frozen. + string = string.newFrozen(); + } + + ByteList bytes = string.getByteList(); + int size = bytes.realSize(); + + RubyIOBuffer buffer = newBuffer(context.runtime, ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size), size, flagsForSize(size)); + + if (isGiven) { + return block.yieldSpecific(context, buffer); + } + + return buffer; } @JRubyMethod(name = "initialize") @@ -385,7 +404,9 @@ public IRubyObject op_cmp(ThreadContext context, IRubyObject other) { @JRubyMethod(name = "resize") public IRubyObject resize(ThreadContext context, IRubyObject size) { - return context.nil; + resize(context, size.convertToInteger().getIntValue()); + + return this; } // MRI: rb_io_buffer_resize @@ -506,7 +527,7 @@ private boolean freeInternal(ThreadContext context) { return false; } - @JRubyMethod(name = "size_of") + @JRubyMethod(name = "size_of", meta = true) public static IRubyObject size_of(ThreadContext context, IRubyObject self, IRubyObject dataType) { if (dataType instanceof RubyArray) { long total = 0; @@ -905,40 +926,58 @@ private static void setValue(ThreadContext context, ByteBuffer buffer, int size, switch (dataType) { case S8: writeByte(context, buffer, offset, (byte) unwrapLong(value)); + return; case U8: writeUnsignedByte(context, buffer, offset, (int) unwrapLong(value)); + return; case u16: writeUnsignedShort(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (int) unwrapLong(value)); + return; case U16: writeUnsignedShort(context, buffer, offset, ByteOrder.BIG_ENDIAN, (int) unwrapLong(value)); + return; case s16: writeShort(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (short) unwrapLong(value)); + return; case S16: writeShort(context, buffer, offset, ByteOrder.BIG_ENDIAN, (short) unwrapLong(value)); + return; case u32: - writeUnsignedInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (long) unwrapLong(value)); + writeUnsignedInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapLong(value)); + return; case U32: - writeUnsignedInt(context, buffer, offset, ByteOrder.BIG_ENDIAN, (long) unwrapLong(value)); + writeUnsignedInt(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapLong(value)); + return; case s32: writeInt(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (int) unwrapLong(value)); + return; case S32: writeInt(context, buffer, offset, ByteOrder.BIG_ENDIAN, (int) unwrapLong(value)); + return; case u64: writeUnsignedLong(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapUnsignedLong(value)); + return; case U64: writeUnsignedLong(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapUnsignedLong(value)); + return; case s64: writeLong(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapLong(value)); + return; case S64: writeLong(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapLong(value)); + return; case f32: writeFloat(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, (float) unwrapDouble(value)); + return; case F32: writeFloat(context, buffer, offset, ByteOrder.BIG_ENDIAN, (float) unwrapDouble(value)); + return; case f64: writeDouble(context, buffer, offset, ByteOrder.LITTLE_ENDIAN, unwrapDouble(value)); + return; case F64: writeDouble(context, buffer, offset, ByteOrder.BIG_ENDIAN, unwrapDouble(value)); + return; } throw context.runtime.newArgumentError("Unknown data_type: " + dataType); // should never happen @@ -1240,7 +1279,7 @@ public IRubyObject op_xor(ThreadContext context, IRubyObject _mask) { return outputBuffer; } - @JRubyMethod(name = "~") + @JRubyMethod(name = "~@") public IRubyObject op_not(ThreadContext context) { ByteBuffer buffer = getBufferForReading(context); @@ -1296,8 +1335,14 @@ public IRubyObject read(ThreadContext context, IRubyObject io, int length, int o } private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBuffer base, int offset, int size) { - int result = OpenFile.readInternal(context, io.openFile, io.openFile.fd(), base, offset, size); - return FiberScheduler.result(context.runtime, result, io.openFile.posix.getErrno().value()); + OpenFile fptr = io.getOpenFileChecked(); + final boolean locked = fptr.lock(); + try { + int result = OpenFile.readInternal(context, fptr, fptr.fd(), base, offset, size); + return FiberScheduler.result(context.runtime, result, fptr.errno()); + } finally { + if (locked) fptr.unlock(); + } } @JRubyMethod(name = "pread") @@ -1366,8 +1411,14 @@ public IRubyObject write(ThreadContext context, IRubyObject io, int length, int } private static IRubyObject writeInternal(ThreadContext context, RubyIO io, ByteBuffer base, int offset, int size) { - int result = OpenFile.writeInternal(context, io.openFile, base, offset, size); - return FiberScheduler.result(context.runtime, result, io.openFile.posix.getErrno().value()); + OpenFile fptr = io.getOpenFileChecked(); + final boolean locked = fptr.lock(); + try { + int result = OpenFile.writeInternal(context, fptr, base, offset, size); + return FiberScheduler.result(context.runtime, result, fptr.errno()); + } finally { + if (locked) fptr.unlock(); + } } @JRubyMethod(name = "pwrite") From a3ae3c36a756ac1157fa92f15908f697672f1056 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 3 Oct 2023 15:37:13 -0500 Subject: [PATCH 15/56] Implement full-buffer boolean operations For test_io_buffer.rb:test_inplace_operators This also necessitated implementing initialize_copy for the dup'ed buffer in the test, along with to_s, inspect, and hexdump for when there's an error condition. --- .../src/main/java/org/jruby/RubyIOBuffer.java | 290 +++++++++++++++++- 1 file changed, 281 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index e5f3f0ab51a..f1f094fe887 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -155,7 +155,7 @@ public void initialize(ThreadContext context, byte[] baseBytes, int size, int fl this.base = base; this.size = size; this.flags = flags; - this.source = source; + this.source = source.isNil() ? null : source; } private static ByteBuffer newBufferBase(Ruby runtime, int size, int flags) { @@ -185,22 +185,166 @@ private static int flagsForSize(int size) { @JRubyMethod(name = "initialize_copy") public IRubyObject initialize_copy(ThreadContext context, IRubyObject other) { - return context.nil; + RubyIOBuffer otherBuffer = (RubyIOBuffer) other; + ByteBuffer sourceBase = otherBuffer.getBufferForReading(context); + int sourceSize = otherBuffer.size; + + initialize(context, null, sourceSize, flagsForSize(size), context.nil); + + return copy(context, otherBuffer, 0, sourceSize, 0); } @JRubyMethod(name = "inspect") public IRubyObject inspect(ThreadContext context) { - return context.nil; + RubyString result = to_s(context); + + if (validate()) { + // Limit the maximum size genearted by inspect. + if (size <= 256) { + hexdump(context, result, 16, base, size, false); + } + } + + return result; + } + + private boolean validate() { + if (source != null) { + return validateSlice(source, base, size); + } + + return true; + } + + private boolean validateSlice(IRubyObject source, ByteBuffer base, int size) { + ByteBuffer sourceBase = null; + int sourceSize = 0; + + if (source instanceof RubyString) { + ByteList sourceBytes = ((RubyString) source).getByteList(); + sourceSize = sourceBytes.getRealSize(); + sourceBase = ByteBuffer.wrap(sourceBytes.getUnsafeBytes(), sourceBytes.begin(), sourceSize); + } else { + RubyIOBuffer sourceBuffer = (RubyIOBuffer) source; + sourceBase = sourceBuffer.base; + sourceSize = sourceBuffer.size; + } + + // Source is invalid: + if (sourceBase == null) return false; + + // Base is out of range: + if (base.hasArray() && sourceBase.hasArray() && base.array() != sourceBase.array()) return false; + + int sourceEnd = sourceSize; + int end = size; + + // End is out of range: + if (end > sourceEnd) return false; + + // It seems okay: + return true; } @JRubyMethod(name = "hexdump") public IRubyObject hexdump(ThreadContext context) { + ByteBuffer base = this.base; + int size = this.size; + if (validate() && base != null) { + RubyString result = RubyString.newStringLight(context.runtime, size * 3 + (size / 16) * 12 + 1); + + hexdump(context, result, 16, base, size, true); + + return result; + } + return context.nil; } + private static RubyString hexdump(ThreadContext context, RubyString string, int width, ByteBuffer base, int size, boolean first) { + byte[] text = new byte[width+1]; + text[width] = '\0'; + + for (int offset = 0; offset < size; offset += width) { + Arrays.fill(text, (byte) 0); + if (first) { + string.cat("0x".getBytes()); + String hex = String.format("0x%08x ", offset); + string.cat(hex.getBytes()); + first = false; + } else { + string.cat(String.format("\n0x%08x ", offset).getBytes()); + } + + for (int i = 0; i < width; i += 1) { + if (offset+i < size) { + int value = Byte.toUnsignedInt(base.get(offset+i)); + + if (value < 127 && value >= 32) { + text[i] = (byte)value; + } + else { + text[i] = '.'; + } + + string.cat(String.format("%02x", value).getBytes()); + } + else { + string.cat(" ".getBytes()); + } + } + + string.cat(' '); + string.cat(text); + } + + return string; + } + @JRubyMethod(name = "to_s") - public IRubyObject to_s(ThreadContext context) { - return context.nil; + public RubyString to_s(ThreadContext context) { + RubyString result = RubyString.newString(context.runtime, "#<"); + + result.append(this.getMetaClass().name(context)); + result.cat(String.format(" %d+%d", System.identityHashCode(base), size).getBytes()); + + if (base == null) { + result.cat(" NULL".getBytes()); + } + + if (isExternal()) { + result.cat(" EXTERNAL".getBytes()); + } + + if (isInternal()) { + result.cat(" INTERNAL".getBytes()); + } + + if (isMapped()) { + result.cat(" MAPPED".getBytes()); + } + + if (isShared()) { + result.cat(" SHARED".getBytes()); + } + + if (isLocked()) { + result.cat(" LOCKED".getBytes()); + } + + if (isReadonly()) { + result.cat(" READONLY".getBytes()); + } + + if (source != null) { + result.cat(" SLICE".getBytes()); + } + + if (!validate()) { + result.cat(" INVALID".getBytes()); + } + + return result.cat(">".getBytes()); } @JRubyMethod(name = "size") @@ -210,7 +354,7 @@ public IRubyObject size(ThreadContext context) { @JRubyMethod(name = "valid?") public IRubyObject valid_p(ThreadContext context) { - return context.nil; + return RubyBoolean.newBoolean(context, validate()); } @JRubyMethod(name = "transfer") @@ -234,7 +378,7 @@ public IRubyObject transfer(ThreadContext context) { private void zero(ThreadContext context) { base = null; size = 0; - source = context.nil; + source = null; } @JRubyMethod(name = "null?") @@ -280,6 +424,10 @@ public IRubyObject shared_p(ThreadContext context) { return newBoolean(context, false); } + private boolean isShared() { + return (flags & SHARED) == SHARED; + } + @JRubyMethod(name = "locked?") public IRubyObject locked_p(ThreadContext context) { return newBoolean(context, isLocked()); @@ -428,6 +576,8 @@ public void resize(ThreadContext context, int size) { ByteBuffer newBase = this.base.isDirect() ? ByteBuffer.allocateDirect(size) : ByteBuffer.allocate(size); newBase.put(this.base); + this.base.clear(); + newBase.clear(); this.base = newBase; this.size = size; @@ -519,7 +669,7 @@ private boolean freeInternal(ThreadContext context) { this.base = null; this.size = 0; this.flags = 0; - this.source = context.nil; + this.source = null; return true; } @@ -1122,8 +1272,8 @@ private void bufferCopy(ThreadContext context, int offset, ByteBuffer sourceBuff } else { destBuffer.position(offset); destBuffer.put(sourceBuffer); - destBuffer.clear(); } + destBuffer.clear(); sourceBuffer.clear(); } @@ -1293,6 +1443,128 @@ public IRubyObject op_not(ThreadContext context) { return outputBuffer; } + @JRubyMethod(name = "and!") + public IRubyObject and_bang(ThreadContext context, IRubyObject _mask) { + if (!(_mask instanceof RubyIOBuffer)) { + throw context.runtime.newTypeError(_mask, context.runtime.getIOBuffer()); + } + + RubyIOBuffer maskData = (RubyIOBuffer) _mask; + + checkMask(context, maskData); + checkOverlaps(context, maskData); + + ByteBuffer base = getBufferForWriting(context); + ByteBuffer maskBase = maskData.getBufferForReading(context); + + bufferAndInPlace(base, size, maskBase, maskData.size); + + return this; + } + + @JRubyMethod(name = "or!") + public IRubyObject or_bang(ThreadContext context, IRubyObject _mask) { + if (!(_mask instanceof RubyIOBuffer)) { + throw context.runtime.newTypeError(_mask, context.runtime.getIOBuffer()); + } + + RubyIOBuffer maskData = (RubyIOBuffer) _mask; + + checkMask(context, maskData); + checkOverlaps(context, maskData); + + ByteBuffer base = getBufferForWriting(context); + ByteBuffer maskBase = maskData.getBufferForReading(context); + + bufferOrInPlace(base, size, maskBase, maskData.size); + + return this; + } + + @JRubyMethod(name = "xor!") + public IRubyObject xor_bang(ThreadContext context, IRubyObject _mask) { + if (!(_mask instanceof RubyIOBuffer)) { + throw context.runtime.newTypeError(_mask, context.runtime.getIOBuffer()); + } + + RubyIOBuffer maskData = (RubyIOBuffer) _mask; + + checkMask(context, maskData); + checkOverlaps(context, maskData); + + ByteBuffer base = getBufferForWriting(context); + ByteBuffer maskBase = maskData.getBufferForReading(context); + + bufferXorInPlace(base, size, maskBase, maskData.size); + + return this; + } + + @JRubyMethod(name = "not!") + public IRubyObject not_bang(ThreadContext context) { + ByteBuffer base = getBufferForWriting(context); + + bufferNotInPlace(base, size); + + return this; + } + + private static void checkMask(ThreadContext context, RubyIOBuffer buffer) { + if (buffer.size == 0) { + throw context.runtime.newBufferMaskError("Zero-length mask given!"); + } + } + + private void checkOverlaps(ThreadContext context, RubyIOBuffer other) { + if (bufferOverlaps(other)) { + throw context.runtime.newBufferMaskError("Mask overlaps source data!"); + } + } + + private boolean bufferOverlaps(RubyIOBuffer other) { + if (base != null && base.hasArray() && other.base != null && other.base.hasArray()) { + if (base.array() == other.base.array()) { + return true; + } + } + + // unsure how to detect overlap for native buffers + return false; + } + + private static void bufferAndInPlace(ByteBuffer a, int aSize, ByteBuffer b, int bSize) { + for (int aIndex = 0; aIndex < aSize;) { + for (int bIndex = 0; aIndex < aSize && bIndex < bSize; bIndex++) { + int curIndex = aIndex++; + a.put(curIndex, (byte) (a.get(curIndex) & b.get(bIndex))); + } + } + } + + private static void bufferOrInPlace(ByteBuffer a, int aSize, ByteBuffer b, int bSize) { + for (int aIndex = 0; aIndex < aSize;) { + for (int bIndex = 0; aIndex < aSize && bIndex < bSize; bIndex++) { + int curIndex = aIndex++; + a.put(curIndex, (byte) (a.get(curIndex) | b.get(bIndex))); + } + } + } + + private static void bufferXorInPlace(ByteBuffer a, int aSize, ByteBuffer b, int bSize) { + for (int aIndex = 0; aIndex < aSize;) { + for (int bIndex = 0; aIndex < aSize && bIndex < bSize; bIndex++) { + int curIndex = aIndex++; + a.put(curIndex, (byte) (a.get(curIndex) ^ b.get(bIndex))); + } + } + } + + private static void bufferNotInPlace(ByteBuffer a, int aSize) { + for (int aIndex = 0; aIndex < aSize; aIndex++) { + a.put(aIndex, (byte) ~a.get(aIndex)); + } + } + @JRubyMethod(name = "read") public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); From 83d26b0884a5daf929eb9f44d8f57982438a69ac Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 3 Oct 2023 15:52:38 -0500 Subject: [PATCH 16/56] Fix endianness when writing values --- .../src/main/java/org/jruby/RubyIOBuffer.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index f1f094fe887..e919891ea9b 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -737,7 +737,10 @@ private static int readUnsignedShort(ThreadContext context, ByteBuffer buffer, i } private static void writeShort(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, short value) { - if (order == ByteOrder.BIG_ENDIAN) buffer.putShort(offset, value); + if (order == ByteOrder.BIG_ENDIAN) { + buffer.putShort(offset, value); + return; + } buffer.putShort(offset, Short.reverseBytes(value)); } @@ -759,7 +762,10 @@ private static long readUnsignedInt(ThreadContext context, ByteBuffer buffer, in } private static void writeInt(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, int value) { - if (order == ByteOrder.BIG_ENDIAN) buffer.putInt(offset, value); + if (order == ByteOrder.BIG_ENDIAN) { + buffer.putInt(offset, value); + return; + } buffer.putInt(offset, Integer.reverseBytes(value)); } @@ -790,7 +796,10 @@ private static BigInteger readUnsignedLong(ThreadContext context, ByteBuffer buf } private static void writeLong(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, long value) { - if (order == ByteOrder.BIG_ENDIAN) buffer.putLong(offset, value); + if (order == ByteOrder.BIG_ENDIAN) { + buffer.putLong(offset, value); + return; + } buffer.putLong(offset, Long.reverseBytes(value)); } @@ -808,7 +817,10 @@ private static float readFloat(ThreadContext context, ByteBuffer buffer, int off } private static void writeFloat(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, float value) { - if (order == ByteOrder.BIG_ENDIAN) buffer.putFloat(offset, value); + if (order == ByteOrder.BIG_ENDIAN) { + buffer.putFloat(offset, value); + return; + } buffer.putFloat(offset, Float.intBitsToFloat(Integer.reverseBytes(Float.floatToIntBits(value)))); } @@ -822,7 +834,10 @@ private static double readDouble(ThreadContext context, ByteBuffer buffer, int o } private static void writeDouble(ThreadContext context, ByteBuffer buffer, int offset, ByteOrder order, double value) { - if (order == ByteOrder.BIG_ENDIAN) buffer.putDouble(offset, value); + if (order == ByteOrder.BIG_ENDIAN) { + buffer.putDouble(offset, value); + return; + } buffer.putDouble(offset, Double.longBitsToDouble(Long.reverseBytes(Double.doubleToLongBits(value)))); } From ad4f3fbfb8bc5dafd8c0309d20e982fa50534307 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 3 Oct 2023 15:54:11 -0500 Subject: [PATCH 17/56] Whitespace --- core/src/main/java/org/jruby/RubyIOBuffer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index e919891ea9b..80d1477e06d 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1691,7 +1691,6 @@ public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject leng public IRubyObject write(ThreadContext context, IRubyObject io, int length, int offset) { validateRange(context, offset, length); - ByteBuffer buffer = getBufferForWriting(context); return writeInternal(context, RubyIO.convertToIO(context, io), buffer, offset, length); From 6ca35e76cab71a44c279b3a48901fa6f67c05211 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 3 Oct 2023 15:57:39 -0500 Subject: [PATCH 18/56] Clear position when putting without offset Moving to Java 13+ would allow us to do absolute puts without the position changing. --- core/src/main/java/org/jruby/RubyIOBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 80d1477e06d..62eb2d47b3b 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1300,8 +1300,8 @@ private void bufferCopy(ThreadContext context, int offset, ByteList sourceBuffer } else { destBuffer.position(offset); destBuffer.put(sourceBuffer.getUnsafeBytes(), sourceBuffer.begin() + sourceOffset, length); - destBuffer.clear(); } + destBuffer.clear(); } @JRubyMethod(name = "get_string") From fad3fad4f9077e2a1ec016a69a128bbbbdbcce61 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Wed, 4 Oct 2023 12:20:40 -0500 Subject: [PATCH 19/56] Reset buffer after read Position is not part of IO::Buffer so we must constantly reset it. --- core/src/main/java/org/jruby/RubyIOBuffer.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 62eb2d47b3b..547e18c910a 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1621,6 +1621,19 @@ public IRubyObject read(ThreadContext context, IRubyObject io, int length, int o return readInternal(context, RubyIO.convertToIO(context, io), buffer, offset, length); } + /** + * Read from the given io into the given buffer base at the given offset and size limit. The buffer will be left + * with its position after the last byte read. + * + * MRI: io_buffer_read_internal + * + * @param context + * @param io + * @param base + * @param offset + * @param size + * @return + */ private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBuffer base, int offset, int size) { OpenFile fptr = io.getOpenFileChecked(); final boolean locked = fptr.lock(); @@ -1628,6 +1641,7 @@ private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBu int result = OpenFile.readInternal(context, fptr, fptr.fd(), base, offset, size); return FiberScheduler.result(context.runtime, result, fptr.errno()); } finally { + base.clear(); if (locked) fptr.unlock(); } } From 8f921b6dc906476d1f361d05e1e5014a01de8803 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Wed, 4 Oct 2023 12:22:08 -0500 Subject: [PATCH 20/56] Only check this MRI constant on MRI --- test/mri/ruby/test_io_buffer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mri/ruby/test_io_buffer.rb b/test/mri/ruby/test_io_buffer.rb index d9fa22fd2a4..1b2160ecc74 100644 --- a/test/mri/ruby/test_io_buffer.rb +++ b/test/mri/ruby/test_io_buffer.rb @@ -333,7 +333,7 @@ def test_read # This is currently a bug in IO:Buffer [#19084] which affects extended # strings. On 32 bit machines, the example below becomes extended, so # we omit this test until the bug is fixed. - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if RUBY_ENGINE == 'ruby' && GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 io = Tempfile.new io.write("Hello World") io.seek(0) From 4693546d2c8e828493e0c0ed8b6c13bd223155ac Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Wed, 4 Oct 2023 12:35:31 -0500 Subject: [PATCH 21/56] Fixes for each_byte --- core/src/main/java/org/jruby/RubyIOBuffer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 547e18c910a..f2d5345bb02 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1046,7 +1046,7 @@ private RubyArray values(ThreadContext context, ByteBuffer buffer, DataType data @JRubyMethod(name = "each_byte") public IRubyObject each_byte(ThreadContext context, Block block) { - if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each"); + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each_byte"); ByteBuffer buffer = getBufferForReading(context); @@ -1055,7 +1055,7 @@ public IRubyObject each_byte(ThreadContext context, Block block) { @JRubyMethod(name = "each_byte") public IRubyObject each_byte(ThreadContext context, IRubyObject _offset, Block block) { - if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", Helpers.arrayOf(_offset)); + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each_byte", Helpers.arrayOf(_offset)); ByteBuffer buffer = getBufferForReading(context); int offset = _offset.convertToInteger().getIntValue(); @@ -1065,7 +1065,7 @@ public IRubyObject each_byte(ThreadContext context, IRubyObject _offset, Block b @JRubyMethod(name = "each_byte") public IRubyObject each_byte(ThreadContext context, IRubyObject _offset, IRubyObject _count, Block block) { - if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each", Helpers.arrayOf(_offset, _count)); + if (!block.isGiven()) return RubyEnumerator.enumeratorize(context.runtime, this, "each_byte", Helpers.arrayOf(_offset, _count)); ByteBuffer buffer = getBufferForReading(context); int offset = _offset.convertToInteger().getIntValue(); @@ -1078,7 +1078,7 @@ private IRubyObject eachByte(ThreadContext context, ByteBuffer buffer, int offse Ruby runtime = context.runtime; for (int i = 0 ; i < count; i++) { - IRubyObject value = wrap(runtime, readByte(context, buffer, offset)); + IRubyObject value = wrap(runtime, readByte(context, buffer, offset + i)); block.yieldSpecific(context, value); } From 687a89f3b8ff83de2b2c611c904f92f2b657a702 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 14:48:05 -0500 Subject: [PATCH 22/56] Implement pread and refactor read/write IO Pushing toward the blessed path always receiving a ByteBuffer, so we can avoid recreating it constantly for small reads. --- .../main/java/org/jruby/FiberScheduler.java | 22 ---- core/src/main/java/org/jruby/RubyIO.java | 60 ++-------- .../src/main/java/org/jruby/RubyIOBuffer.java | 109 +++++++++++++++++- core/src/main/java/org/jruby/RubyThread.java | 7 +- .../main/java/org/jruby/util/io/OpenFile.java | 108 ++++++++--------- .../java/org/jruby/util/io/PosixShim.java | 2 + 6 files changed, 171 insertions(+), 137 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index 4b5b29aae1f..0a2f03c61f4 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -96,17 +96,6 @@ public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, } // MRI: rb_fiber_scheduler_io_read_memory - public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, byte[] base, int size, int length) { - RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, ByteBuffer.wrap(base), size, RubyIOBuffer.LOCKED); - - IRubyObject result = ioRead(context, scheduler, io, buffer, length); - - buffer.unlock(context); - buffer.free(context); - - return result; - } - public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int size, int length) { RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED); @@ -119,17 +108,6 @@ public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject schedu } // MRI: rb_fiber_scheduler_io_write_memory - public static IRubyObject ioWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, byte[] base, int size, int length) { - RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, ByteBuffer.wrap(base), size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); - - IRubyObject result = ioWrite(context, scheduler, io, buffer, length); - - buffer.unlock(context); - buffer.free(context); - - return result; - } - public static IRubyObject ioWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int size, int length) { RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java index 9556315d3dc..ac6c52a6fb7 100644 --- a/core/src/main/java/org/jruby/RubyIO.java +++ b/core/src/main/java/org/jruby/RubyIO.java @@ -4797,14 +4797,14 @@ public IRubyObject pread(ThreadContext context, IRubyObject len, IRubyObject off } @JRubyMethod(name = "pread") - public IRubyObject pread(ThreadContext context, IRubyObject len, IRubyObject offset, IRubyObject str) { + public IRubyObject pread(ThreadContext context, IRubyObject _length, IRubyObject _from, IRubyObject str) { Ruby runtime = context.runtime; - int count = len.convertToInteger().getIntValue(); - long off = offset.convertToInteger().getIntValue(); + int length = _length.convertToInteger().getIntValue(); + int from = _from.convertToInteger().getIntValue(); - RubyString string = EncodingUtils.setStrBuf(runtime, str, count); - if (count == 0) return string; + RubyString string = EncodingUtils.setStrBuf(runtime, str, length); + if (length == 0) return string; OpenFile fptr = getOpenFile(); fptr.checkByteReadable(context); @@ -4813,54 +4813,14 @@ public IRubyObject pread(ThreadContext context, IRubyObject len, IRubyObject off fptr.checkClosed(); ByteList strByteList = string.getByteList(); + int read; - try { - return context.getThread().executeTaskBlocking(context, fd, new RubyThread.Task() { - @Override - public IRubyObject run(ThreadContext context, ChannelFD channelFD) throws InterruptedException { - Ruby runtime = context.runtime; - - ByteBuffer wrap = ByteBuffer.wrap(strByteList.unsafeBytes(), strByteList.begin(), count); - int read = 0; - - try { - if (fd.chFile != null) { - read = fd.chFile.read(wrap, off); + ByteBuffer wrap = ByteBuffer.wrap(strByteList.unsafeBytes(), strByteList.begin(), length); + read = OpenFile.preadInternal(context, fd, wrap, from, length); - if (read == -1) { - throw runtime.newEOFError(); - } - } else if (fd.chNative != null) { - read = (int) runtime.getPosix().pread(fd.chNative.getFD(), wrap, count, off); + string.setReadLength(read); - if (read == 0) { - throw runtime.newEOFError(); - } else if (read == -1) { - throw runtime.newErrnoFromInt(runtime.getPosix().errno()); - } - } else if (fd.chRead != null) { - read = fd.chRead.read(wrap); - } else { - throw runtime.newIOError("not opened for reading"); - } - } catch (IOException ioe) { - throw Helpers.newIOErrorFromException(runtime, ioe); - } - - string.setReadLength(read); - - return string; - } - - @Override - public void wakeup(RubyThread thread, ChannelFD channelFD) { - // FIXME: NO! This will kill many native channels. Must be nonblocking to interrupt. - thread.getNativeThread().interrupt(); - } - }); - } catch (InterruptedException ie) { - throw context.runtime.newConcurrencyError("IO operation interrupted"); - } + return string; } @JRubyMethod(name = "pwrite") diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index f2d5345bb02..0d8ad39da40 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1647,15 +1647,32 @@ private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBu } @JRubyMethod(name = "pread") - public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length) { - return context.nil; + public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject _from) { + int from = RubyNumeric.num2int(_from); + + int offset = 0; + int length = defaultLength(context, offset); + + return pread(context, RubyIO.convertToIO(context, io), from, length, offset); + } + + @JRubyMethod(name = "pread") + public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject _from, IRubyObject _length) { + int from = RubyNumeric.num2int(_from); + + int offset = 0; + int length = extractLength(context, _length, offset); + + return pread(context, RubyIO.convertToIO(context, io), from, length, offset); } - @JRubyMethod(name = "pread", required = 1, optional = 3, checkArity = false) + @JRubyMethod(name = "pread", required = 2, optional = 2, checkArity = false) public IRubyObject pread(ThreadContext context, IRubyObject[] args) { - Arity.checkArgumentCount(context, args, 3, 4); + Arity.checkArgumentCount(context, args, 2, 4); switch (args.length) { + case 2: + return pread(context, args[0], args[1]); case 3: return pread(context, args[0], args[1], args[2]); case 4: @@ -1665,8 +1682,88 @@ public IRubyObject pread(ThreadContext context, IRubyObject[] args) { return context.nil; } - public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length, IRubyObject offset) { - throw context.runtime.newNotImplementedError("pread"); + public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject _from, IRubyObject _length, IRubyObject _offset) { + int from = RubyNumeric.num2int(_from); + + int offset = extractOffset(context, _offset); + int length = extractLength(context, _length, offset); + + return pread(context, RubyIO.convertToIO(context, io), from, length, offset); + } + + public IRubyObject pread(ThreadContext context, RubyIO io, int from, int length, int offset) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPRead(context, scheduler, io, this, length, offset); + + if (result != UNDEF) { + return result; + } + } + + validateRange(context, offset, length); + + ByteBuffer buffer = getBufferForWriting(context); + + return preadInternal(context, io, buffer, from, offset, length); + } + + /** + * Read from the given io into the given buffer base at the given offset, from location, and size limit. The buffer will be left + * with its position unchanged. + * + * MRI: io_buffer_pread_internal + * + * @param context + * @param io + * @param base + * @param from + * @param offset + * @param size + * @return + */ + private static IRubyObject preadInternal(ThreadContext context, RubyIO io, ByteBuffer base, int from, int offset, int size) { + OpenFile fptr = io.getOpenFileChecked(); + final boolean locked = fptr.lock(); + try { + base.position(offset); + int result = OpenFile.preadInternal(context, fptr.fd(), base, from, size); + return FiberScheduler.result(context.runtime, result, fptr.errno()); + } finally { + base.clear(); + if (locked) fptr.unlock(); + } + } + + // MRI: length parts of io_buffer_extract_length_offset and io_buffer_extract_length + private int extractLength(ThreadContext context, IRubyObject _length, int offset) { + if (!_length.isNil()) { + if (RubyNumeric.negativeInt(context, _length)) { + throw context.runtime.newArgumentError("Length can't be negative!"); + } + + return RubyNumeric.num2int(_length); + } + + return defaultLength(context, offset); + } + + private int defaultLength(ThreadContext context, int offset) { + if (offset > size) { + throw context.runtime.newArgumentError("The given offset is bigger than the buffer size!"); + } + + // Note that the "length" is computed by the size the offset. + return size - offset; + } + + // MRI: offset parts of io_buffer_extract_length_offset and io_buffer_extract_offset + private int extractOffset(ThreadContext context, IRubyObject _offset) { + if (RubyNumeric.negativeInt(context, _offset)) { + throw context.runtime.newArgumentError("Offset can't be negative!"); + } + + return RubyNumeric.num2int(_offset); } @JRubyMethod(name = "write") diff --git a/core/src/main/java/org/jruby/RubyThread.java b/core/src/main/java/org/jruby/RubyThread.java index 63b6443a391..7f0bf5c421d 100644 --- a/core/src/main/java/org/jruby/RubyThread.java +++ b/core/src/main/java/org/jruby/RubyThread.java @@ -1830,7 +1830,12 @@ private void preReadWrite(ThreadContext context, Data data, ReadWrite extends Unblocker { - public int run(ThreadContext context, Data data, byte[] bytes, int start, int length) throws InterruptedException; + /** + * @deprecated Prefer version that receives ByteBuffer rather than recreating every time. + */ + public default int run(ThreadContext context, Data data, byte[] bytes, int start, int length) throws InterruptedException { + return run(context, data, ByteBuffer.wrap(bytes), start, length); + } public int run(ThreadContext context, Data data, ByteBuffer bytes, int start, int length) throws InterruptedException; public void wakeup(RubyThread thread, Data data); } diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index da2462e5e22..5122cbf9090 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -42,6 +42,7 @@ import org.jruby.ext.fcntl.FcntlLibrary; import org.jruby.platform.Platform; import org.jruby.runtime.Block; +import org.jruby.runtime.Helpers; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.ByteList; @@ -1333,16 +1334,6 @@ && waitReadable(context, fd)) { } final static RubyThread.ReadWrite READ_TASK = new RubyThread.ReadWrite() { - @Override - public int run(ThreadContext context, OpenFile fptr, byte[] buffer, int start, int length) throws InterruptedException { - ChannelFD fd = preRead(context, fptr); - try { - return fptr.posix.read(fd, buffer, start, length, fptr.nonblock); - } finally { - fptr.lock(); - } - } - @Override public int run(ThreadContext context, OpenFile fptr, ByteBuffer buffer, int start, int length) throws InterruptedException { ChannelFD fd = preRead(context, fptr); @@ -1374,20 +1365,12 @@ public void wakeup(RubyThread thread, OpenFile data) { }; final static RubyThread.ReadWrite WRITE_TASK = new RubyThread.ReadWrite() { - @Override - public int run(ThreadContext context, OpenFile fptr, byte[] bytes, int start, int length) throws InterruptedException { - ChannelFD fd = preWrite(context, fptr); - try { - return fptr.posix.write(fd, bytes, start, length, fptr.nonblock); - } finally { - fptr.lock(); - } - } - @Override public int run(ThreadContext context, OpenFile fptr, ByteBuffer bytes, int start, int length) throws InterruptedException { ChannelFD fd = preWrite(context, fptr); try { + bytes.position(start); + bytes.limit(start + length); return fptr.posix.write(fd, bytes, start, length, fptr.nonblock); } finally { fptr.lock(); @@ -1415,36 +1398,50 @@ public void wakeup(RubyThread thread, OpenFile data) { } }; - // rb_read_internal, rb_io_read_memory, rb_io_buffer_read_internal - public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, byte[] bufBytes, int buf, int count) { - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + public static final RubyThread.ReadWrite PREAD_TASK = new RubyThread.ReadWrite() { + @Override + public int run(ThreadContext context, ChannelFD fd, ByteBuffer bytes, int from, int length) throws InterruptedException { + Ruby runtime = context.runtime; - if (result != RubyBasicObject.UNDEF) { - FiberScheduler.resultApply(context, result); - } - } - // if we can do selection and this is not a non-blocking call, do selection + int read = 0; - /* - NOTE CON: We only do this selection because on the JDK, blocking calls to NIO channels can't be - interrupted, and we need to be able to interrupt blocking reads. In MRI, this logic is always just a - simple read(2) because EINTR does not damage the descriptor. + try { + if (fd.chFile != null) { + read = fd.chFile.read(bytes, from); - Doing selects here on ENXIO native channels caused FIFOs to block for read all the time, even when no - writers are connected. This may or may not be correct behavior for selects against FIFOs, but in any - case since MRI does not do a select here at all I believe correct logic is to skip the select when - working with any native descriptor. - */ + if (read == -1) { + throw runtime.newEOFError(); + } + } else if (fd.chNative != null) { + read = (int) runtime.getPosix().pread(fd.chNative.getFD(), bytes, length, from); - preRead(context, fptr, fd); + if (read == 0) { + throw runtime.newEOFError(); + } else if (read == -1) { + throw runtime.newErrnoFromInt(runtime.getPosix().errno()); + } + } else if (fd.chRead != null) { + read = fd.chRead.read(bytes); + } else { + throw runtime.newIOError("not opened for reading"); + } - try { - return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, READ_TASK); - } catch (InterruptedException ie) { - throw context.runtime.newConcurrencyError("IO operation interrupted"); + return read; + } catch (IOException ioe) { + throw Helpers.newIOErrorFromException(runtime, ioe); + } } + + @Override + public void wakeup(RubyThread thread, ChannelFD channelFD) { + // FIXME: NO! This will kill many native channels. Must be nonblocking to interrupt. + thread.getNativeThread().interrupt(); + } + }; + + // rb_read_internal, rb_io_read_memory, rb_io_buffer_read_internal + public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, byte[] bufBytes, int buf, int count) { + return readInternal(context, fptr, fd, ByteBuffer.wrap(bufBytes), buf, count); } // rb_io_buffer_read_internal @@ -1479,6 +1476,14 @@ simple read(2) because EINTR does not damage the descriptor. } } + public static int preadInternal(ThreadContext context, ChannelFD fd, ByteBuffer buffer, int from, int length) { + try { + return context.getThread().executeReadWrite(context, fd, buffer, from, length, PREAD_TASK); + } catch (InterruptedException ie) { + throw context.runtime.newConcurrencyError("IO operation interrupted"); + } + } + private static void preRead(ThreadContext context, OpenFile fptr, ChannelFD fd) { if (fd == null) { // stream was closed on its way in, raise appropriate error @@ -2402,20 +2407,7 @@ static int binwriteString(OpenFile fptr, byte[] bytes, int start, int length) { // rb_write_internal, rb_io_write_memory public static int writeInternal(ThreadContext context, OpenFile fptr, byte[] bufBytes, int buf, int count) { - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); - - if (result != RubyBasicObject.UNDEF) { - FiberScheduler.resultApply(context, result); - } - } - - try { - return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, WRITE_TASK); - } catch (InterruptedException ie) { - throw context.runtime.newConcurrencyError("IO operation interrupted"); - } + return writeInternal(context, fptr, ByteBuffer.wrap(bufBytes), buf, count); } // rb_io_buffer_write_internal diff --git a/core/src/main/java/org/jruby/util/io/PosixShim.java b/core/src/main/java/org/jruby/util/io/PosixShim.java index ed3c36cba3a..45c7ccbc154 100644 --- a/core/src/main/java/org/jruby/util/io/PosixShim.java +++ b/core/src/main/java/org/jruby/util/io/PosixShim.java @@ -151,6 +151,8 @@ public int read(ChannelFD fd, ByteBuffer buffer, int offset, int length, boolean try { if (checkForBlocking(fd, nonblock)) return -1; + buffer.position(offset); + buffer.limit(offset + length); return performRead(fd, buffer, nonblock); } catch (IOException ioe) { setErrno(Helpers.errnoFromException(ioe)); From 04f0a0804838a7a959b9d44c469dfbe8e58a0067 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:22:46 -0500 Subject: [PATCH 23/56] Implement pwrite and tweak pread --- .../main/java/org/jruby/FiberScheduler.java | 16 +- core/src/main/java/org/jruby/RubyIO.java | 43 +---- .../src/main/java/org/jruby/RubyIOBuffer.java | 165 +++++++++++++++--- .../main/java/org/jruby/util/io/OpenFile.java | 43 ++++- 4 files changed, 201 insertions(+), 66 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index 0a2f03c61f4..3a5f0d180f7 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -69,12 +69,12 @@ public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, I } // MRI: rb_fiber_scheduler_io_pread - public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { - return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); + public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int from, int length, int offset) { + return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, context.runtime.newFixnum(from), context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); } - public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, IRubyObject length, IRubyObject offset) { - return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, length, offset); + public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger from, RubyInteger length, RubyInteger offset) { + return Helpers.invokeChecked(context, scheduler, "io_pread", io, buffer, from, length, offset); } // MRI: rb_fiber_scheduler_io_write @@ -87,12 +87,12 @@ public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, } // MRI: rb_fiber_scheduler_io_pwrite - public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { - return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); + public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int from, int length, int offset) { + return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, context.runtime.newFixnum(from), context.runtime.newFixnum(length), context.runtime.newFixnum(offset)); } - public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, IRubyObject length, IRubyObject offset) { - return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, length, offset); + public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger from, RubyInteger length, RubyInteger offset) { + return Helpers.invokeChecked(context, scheduler, "io_pwrite", io, buffer, from, length, offset); } // MRI: rb_fiber_scheduler_io_read_memory diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java index ac6c52a6fb7..7c03fd1679c 100644 --- a/core/src/main/java/org/jruby/RubyIO.java +++ b/core/src/main/java/org/jruby/RubyIO.java @@ -4835,7 +4835,7 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject str, IRubyObject of string = str.convertToString(); } - long off = offset.convertToInteger().getLongValue(); + int off = offset.convertToInteger().getIntValue(); RubyIO io = GetWriteIO(); fptr = io.getOpenFile(); @@ -4846,41 +4846,14 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject str, IRubyObject of ByteList strByteList = buf.getByteList(); - try { - return context.getThread().executeTaskBlocking(context, fd, new RubyThread.Task() { - @Override - public IRubyObject run(ThreadContext context, ChannelFD channelFD) throws InterruptedException { - Ruby runtime = context.runtime; - - int length = strByteList.realSize(); - ByteBuffer wrap = ByteBuffer.wrap(strByteList.unsafeBytes(), strByteList.begin(), length); - int written = 0; - - try { - if (fd.chFile != null) { - written = fd.chFile.write(wrap, off); - } else if (fd.chNative != null) { - written = (int) runtime.getPosix().pwrite(fd.chNative.getFD(), wrap, length, off); - } else if (fd.chWrite != null) { - written = fd.chWrite.write(wrap); - } else { - throw runtime.newIOError("not opened for writing"); - } - } catch (IOException ioe) { - throw Helpers.newIOErrorFromException(runtime, ioe); - } - return runtime.newFixnum(written); - } + int length = strByteList.realSize(); + ByteBuffer wrap = ByteBuffer.wrap(strByteList.unsafeBytes(), strByteList.begin(), length); - @Override - public void wakeup(RubyThread thread, ChannelFD channelFD) { - // FIXME: NO! This will kill many native channels. Must be nonblocking to interrupt. - thread.getNativeThread().interrupt(); - } - }); - } catch (InterruptedException ie) { - throw context.runtime.newConcurrencyError("IO operation interrupted"); - } + int written; + + written = OpenFile.pwriteInternal(context, fd, wrap, off, length); + + return context.runtime.newFixnum(written); } /** diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 0d8ad39da40..115001bbae1 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1638,6 +1638,8 @@ private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBu OpenFile fptr = io.getOpenFileChecked(); final boolean locked = fptr.lock(); try { + base.position(offset); + base.limit(offset + size); int result = OpenFile.readInternal(context, fptr, fptr.fd(), base, offset, size); return FiberScheduler.result(context.runtime, result, fptr.errno()); } finally { @@ -1648,20 +1650,44 @@ private static IRubyObject readInternal(ThreadContext context, RubyIO io, ByteBu @JRubyMethod(name = "pread") public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject _from) { - int from = RubyNumeric.num2int(_from); - + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger fromInteger = _from.convertToInteger(); int offset = 0; int length = defaultLength(context, offset); + if (!scheduler.isNil()) { + Ruby runtime = context.runtime; + IRubyObject result = FiberScheduler.ioPRead(context, scheduler, io, this, fromInteger, RubyFixnum.newFixnum(runtime, length), RubyFixnum.zero(runtime)); + + if (result != UNDEF) { + return result; + } + } + + int from = RubyNumeric.num2int(_from); + + return pread(context, RubyIO.convertToIO(context, io), from, length, offset); } @JRubyMethod(name = "pread") public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject _from, IRubyObject _length) { - int from = RubyNumeric.num2int(_from); + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger fromInteger = _from.convertToInteger(); + RubyInteger lengthInteger = _length.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPRead(context, scheduler, io, this, fromInteger, lengthInteger, RubyFixnum.zero(context.runtime)); + + if (result != UNDEF) { + return result; + } + } + + int from = RubyNumeric.num2int(fromInteger); int offset = 0; - int length = extractLength(context, _length, offset); + int length = extractLength(context, lengthInteger, offset); return pread(context, RubyIO.convertToIO(context, io), from, length, offset); } @@ -1683,24 +1709,28 @@ public IRubyObject pread(ThreadContext context, IRubyObject[] args) { } public IRubyObject pread(ThreadContext context, IRubyObject io, IRubyObject _from, IRubyObject _length, IRubyObject _offset) { - int from = RubyNumeric.num2int(_from); - - int offset = extractOffset(context, _offset); - int length = extractLength(context, _length, offset); - - return pread(context, RubyIO.convertToIO(context, io), from, length, offset); - } - - public IRubyObject pread(ThreadContext context, RubyIO io, int from, int length, int offset) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger fromInteger = _from.convertToInteger(); + RubyInteger lengthInteger = _length.convertToInteger(); + RubyInteger offsetInteger = _offset.convertToInteger(); + if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioPRead(context, scheduler, io, this, length, offset); + IRubyObject result = FiberScheduler.ioPRead(context, scheduler, io, this, fromInteger, lengthInteger, offsetInteger); if (result != UNDEF) { return result; } } + int from = RubyNumeric.num2int(fromInteger); + + int offset = extractOffset(context, offsetInteger); + int length = extractLength(context, lengthInteger, offset); + + return pread(context, RubyIO.convertToIO(context, io), from, length, offset); + } + + public IRubyObject pread(ThreadContext context, RubyIO io, int from, int length, int offset) { validateRange(context, offset, length); ByteBuffer buffer = getBufferForWriting(context); @@ -1727,6 +1757,7 @@ private static IRubyObject preadInternal(ThreadContext context, RubyIO io, ByteB final boolean locked = fptr.lock(); try { base.position(offset); + base.limit(offset + size); int result = OpenFile.preadInternal(context, fptr.fd(), base, from, size); return FiberScheduler.result(context.runtime, result, fptr.errno()); } finally { @@ -1758,7 +1789,7 @@ private int defaultLength(ThreadContext context, int offset) { } // MRI: offset parts of io_buffer_extract_length_offset and io_buffer_extract_offset - private int extractOffset(ThreadContext context, IRubyObject _offset) { + private static int extractOffset(ThreadContext context, IRubyObject _offset) { if (RubyNumeric.negativeInt(context, _offset)) { throw context.runtime.newArgumentError("Offset can't be negative!"); } @@ -1766,6 +1797,14 @@ private int extractOffset(ThreadContext context, IRubyObject _offset) { return RubyNumeric.num2int(_offset); } + private static int extractSize(ThreadContext context, IRubyObject _size) { + if (RubyNumeric.negativeInt(context, _size)) { + throw context.runtime.newArgumentError("Size can't be negative!"); + } + + return RubyNumeric.num2int(_size); + } + @JRubyMethod(name = "write") public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject length) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); @@ -1811,23 +1850,66 @@ private static IRubyObject writeInternal(ThreadContext context, RubyIO io, ByteB OpenFile fptr = io.getOpenFileChecked(); final boolean locked = fptr.lock(); try { + base.position(offset); + base.limit(offset + size); int result = OpenFile.writeInternal(context, fptr, base, offset, size); return FiberScheduler.result(context.runtime, result, fptr.errno()); } finally { + base.clear(); if (locked) fptr.unlock(); } } @JRubyMethod(name = "pwrite") - public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length) { - return context.nil; + public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject _from) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger fromInteger = _from.convertToInteger(); + int offset = 0; + int length = defaultLength(context, offset); + + if (!scheduler.isNil()) { + Ruby runtime = context.runtime; + IRubyObject result = FiberScheduler.ioPWrite(context, scheduler, io, this, fromInteger, RubyFixnum.newFixnum(runtime, length), RubyFixnum.zero(runtime)); + + if (result != UNDEF) { + return result; + } + } + + int from = RubyNumeric.num2int(fromInteger); + + return pwrite(context, RubyIO.convertToIO(context, io), from, length, offset); + } + + @JRubyMethod(name = "pwrite") + public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject _from, IRubyObject _length) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger fromInteger = _from.convertToInteger(); + RubyInteger lengthInteger = _length.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPWrite(context, scheduler, io, this, fromInteger, lengthInteger, RubyFixnum.zero(context.runtime)); + + if (result != UNDEF) { + return result; + } + } + + int from = RubyNumeric.num2int(fromInteger); + + int offset = 0; + int length = extractLength(context, lengthInteger, offset); + + return pwrite(context, RubyIO.convertToIO(context, io), from, length, offset); } - @JRubyMethod(name = "pwrite", required = 1, optional = 3, checkArity = false) + @JRubyMethod(name = "pwrite", required = 2, optional = 2, checkArity = false) public IRubyObject pwrite(ThreadContext context, IRubyObject[] args) { - Arity.checkArgumentCount(context, args, 3, 4); + Arity.checkArgumentCount(context, args, 2, 4); switch (args.length) { + case 2: + return pwrite(context, args[0], args[1]); case 3: return pwrite(context, args[0], args[1], args[2]); case 4: @@ -1837,8 +1919,49 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject[] args) { return context.nil; } - public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject from, IRubyObject length, IRubyObject offset) { - return context.nil; + public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject _from, IRubyObject _length, IRubyObject _offset) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger fromInteger = _from.convertToInteger(); + RubyInteger lengthInteger = _length.convertToInteger(); + RubyInteger offsetInteger = _offset.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPWrite(context, scheduler, io, this, fromInteger, lengthInteger, offsetInteger); + + if (result != UNDEF) { + return result; + } + } + + int from = RubyNumeric.num2int(fromInteger); + + int offset = extractOffset(context, offsetInteger); + int length = extractLength(context, lengthInteger, offset); + + return pwrite(context, RubyIO.convertToIO(context, io), from, length, offset); + } + + public IRubyObject pwrite(ThreadContext context, RubyIO io, int from, int length, int offset) { + validateRange(context, offset, length); + + ByteBuffer buffer = getBufferForReading(context); + int size = this.size - offset; + + return pwriteInternal(context, RubyIO.convertToIO(context, io), buffer, from, offset, length); + } + + private static IRubyObject pwriteInternal(ThreadContext context, RubyIO io, ByteBuffer base, int from, int offset, int size) { + OpenFile fptr = io.getOpenFileChecked(); + final boolean locked = fptr.lock(); + try { + base.position(offset); + base.limit(offset + size); + int result = OpenFile.pwriteInternal(context, fptr.fd(), base, from, size); + return FiberScheduler.result(context.runtime, result, fptr.errno()); + } finally { + base.clear(); + if (locked) fptr.unlock(); + } } enum DataType { diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index 5122cbf9090..600f6cba46d 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -1439,9 +1439,38 @@ public void wakeup(RubyThread thread, ChannelFD channelFD) { } }; + public static final RubyThread.ReadWrite PWRITE_TASK = new RubyThread.ReadWrite() { + @Override + public int run(ThreadContext context, ChannelFD fd, ByteBuffer bytes, int from, int length) throws InterruptedException { + Ruby runtime = context.runtime; + int written = 0; + + try { + if (fd.chFile != null) { + written = fd.chFile.write(bytes, from); + } else if (fd.chNative != null) { + written = (int) runtime.getPosix().pwrite(fd.chNative.getFD(), bytes, length, from); + } else if (fd.chWrite != null) { + written = fd.chWrite.write(bytes); + } else { + throw runtime.newIOError("not opened for writing"); + } + } catch (IOException ioe) { + throw Helpers.newIOErrorFromException(runtime, ioe); + } + return written; + } + + @Override + public void wakeup(RubyThread thread, ChannelFD channelFD) { + // FIXME: NO! This will kill many native channels. Must be nonblocking to interrupt. + thread.getNativeThread().interrupt(); + } + }; + // rb_read_internal, rb_io_read_memory, rb_io_buffer_read_internal public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, byte[] bufBytes, int buf, int count) { - return readInternal(context, fptr, fd, ByteBuffer.wrap(bufBytes), buf, count); + return readInternal(context, fptr, fd, ByteBuffer.wrap(bufBytes, buf, count), buf, count); } // rb_io_buffer_read_internal @@ -1484,6 +1513,16 @@ public static int preadInternal(ThreadContext context, ChannelFD fd, ByteBuffer } } + public static int pwriteInternal(ThreadContext context, ChannelFD fd, ByteBuffer bytes, int from, int length) { + int written; + try { + written = context.getThread().executeReadWrite(context, fd, bytes, from, length, PWRITE_TASK); + } catch (InterruptedException ie) { + throw context.runtime.newConcurrencyError("IO operation interrupted"); + } + return written; + } + private static void preRead(ThreadContext context, OpenFile fptr, ChannelFD fd) { if (fd == null) { // stream was closed on its way in, raise appropriate error @@ -2407,7 +2446,7 @@ static int binwriteString(OpenFile fptr, byte[] bytes, int start, int length) { // rb_write_internal, rb_io_write_memory public static int writeInternal(ThreadContext context, OpenFile fptr, byte[] bufBytes, int buf, int count) { - return writeInternal(context, fptr, ByteBuffer.wrap(bufBytes), buf, count); + return writeInternal(context, fptr, ByteBuffer.wrap(bufBytes, buf, count), buf, count); } // rb_io_buffer_write_internal From 86aecf2db7e3762266248e5c6ec9a29a0186faca Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:23:10 -0500 Subject: [PATCH 24/56] Add zero-arity get_string --- core/src/main/java/org/jruby/RubyIOBuffer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 115001bbae1..511b029bc7d 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1304,6 +1304,11 @@ private void bufferCopy(ThreadContext context, int offset, ByteList sourceBuffer destBuffer.clear(); } + @JRubyMethod(name = "get_string") + public IRubyObject get_string(ThreadContext context) { + return getString(context, 0, size, ASCIIEncoding.INSTANCE); + } + @JRubyMethod(name = "get_string") public IRubyObject get_string(ThreadContext context, IRubyObject _offset) { int offset = RubyNumeric.num2int(_offset); From 11ef245ab2030bb5b9cf6e492f14841d74220dab Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:23:39 -0500 Subject: [PATCH 25/56] Reimpl bitwise ops based on latest MRI --- .../src/main/java/org/jruby/RubyIOBuffer.java | 87 +++++++++---------- 1 file changed, 40 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 511b029bc7d..da9d1b8e7fd 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1403,62 +1403,48 @@ public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyO @JRubyMethod(name = "&") public IRubyObject op_and(ThreadContext context, IRubyObject _mask) { - ByteBuffer buffer = getBufferForReading(context); + RubyIOBuffer maskBuffer = (RubyIOBuffer) _mask; - int mask = _mask.convertToInteger().getIntValue(); + checkMask(context, maskBuffer); - RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); - ByteBuffer output = outputBuffer.base; + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flagsForSize(size)); - for (int i = 0; i < buffer.capacity(); i++) { - output.put(i, (byte) (buffer.get(i) & mask)); - } + bufferAnd(outputBuffer.base, base, size, maskBuffer.base, maskBuffer.size); return outputBuffer; } @JRubyMethod(name = "|") public IRubyObject op_or(ThreadContext context, IRubyObject _mask) { - ByteBuffer buffer = getBufferForReading(context); + RubyIOBuffer maskBuffer = (RubyIOBuffer) _mask; - int mask = _mask.convertToInteger().getIntValue(); + checkMask(context, maskBuffer); - RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); - ByteBuffer output = outputBuffer.base; + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flagsForSize(size)); - for (int i = 0; i < buffer.capacity(); i++) { - output.put(i, (byte) (buffer.get(i) | mask)); - } + bufferOr(outputBuffer.base, base, size, maskBuffer.base, maskBuffer.size); return outputBuffer; } @JRubyMethod(name = "^") public IRubyObject op_xor(ThreadContext context, IRubyObject _mask) { - ByteBuffer buffer = getBufferForReading(context); + RubyIOBuffer maskBuffer = (RubyIOBuffer) _mask; - int mask = _mask.convertToInteger().getIntValue(); + checkMask(context, maskBuffer); - RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); - ByteBuffer output = outputBuffer.base; + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flagsForSize(size)); - for (int i = 0; i < buffer.capacity(); i++) { - output.put(i, (byte) (buffer.get(i) ^ mask)); - } + bufferXor(outputBuffer.base, base, size, maskBuffer.base, maskBuffer.size); return outputBuffer; } - @JRubyMethod(name = "~@") + @JRubyMethod(name = "~") public IRubyObject op_not(ThreadContext context) { - ByteBuffer buffer = getBufferForReading(context); - - RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flags); - ByteBuffer output = outputBuffer.base; + RubyIOBuffer outputBuffer = newBuffer(context.runtime, size, flagsForSize(size)); - for (int i = 0; i < buffer.capacity(); i++) { - output.put(i, (byte) ~buffer.get(i)); - } + bufferNot(outputBuffer.base, base, size); return outputBuffer; } @@ -1552,37 +1538,44 @@ private boolean bufferOverlaps(RubyIOBuffer other) { return false; } + private static void bufferAnd(ByteBuffer output, ByteBuffer base, int size, ByteBuffer mask, int maskSize) { + for (int offset = 0; offset < size; offset += 1) { + output.put(offset, (byte) (base.get(offset) & mask.get(offset % maskSize))); + } + } + private static void bufferAndInPlace(ByteBuffer a, int aSize, ByteBuffer b, int bSize) { - for (int aIndex = 0; aIndex < aSize;) { - for (int bIndex = 0; aIndex < aSize && bIndex < bSize; bIndex++) { - int curIndex = aIndex++; - a.put(curIndex, (byte) (a.get(curIndex) & b.get(bIndex))); - } + bufferAnd(a, a, aSize, b, bSize); + } + + private static void bufferOr(ByteBuffer output, ByteBuffer base, int size, ByteBuffer mask, int maskSize) { + for (int offset = 0; offset < size; offset += 1) { + output.put(offset, (byte) (base.get(offset) | mask.get(offset % maskSize))); } } private static void bufferOrInPlace(ByteBuffer a, int aSize, ByteBuffer b, int bSize) { - for (int aIndex = 0; aIndex < aSize;) { - for (int bIndex = 0; aIndex < aSize && bIndex < bSize; bIndex++) { - int curIndex = aIndex++; - a.put(curIndex, (byte) (a.get(curIndex) | b.get(bIndex))); - } + bufferOr(a, a, aSize, b, bSize); + } + + private static void bufferXor(ByteBuffer output, ByteBuffer base, int size, ByteBuffer mask, int maskSize) { + for (int offset = 0; offset < size; offset += 1) { + output.put(offset, (byte) (base.get(offset) ^ mask.get(offset % maskSize))); } } private static void bufferXorInPlace(ByteBuffer a, int aSize, ByteBuffer b, int bSize) { - for (int aIndex = 0; aIndex < aSize;) { - for (int bIndex = 0; aIndex < aSize && bIndex < bSize; bIndex++) { - int curIndex = aIndex++; - a.put(curIndex, (byte) (a.get(curIndex) ^ b.get(bIndex))); - } + bufferXor(a, a, aSize, b, bSize); + } + + private static void bufferNot(ByteBuffer output, ByteBuffer base, int size) { + for (int offset = 0; offset < size; offset += 1) { + output.put(offset, (byte) ~base.get(offset)); } } private static void bufferNotInPlace(ByteBuffer a, int aSize) { - for (int aIndex = 0; aIndex < aSize; aIndex++) { - a.put(aIndex, (byte) ~a.get(aIndex)); - } + bufferNot(a, a, aSize); } @JRubyMethod(name = "read") From fcf78abdb0c6218807878f2e8f488045c04eef21 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:24:33 -0500 Subject: [PATCH 26/56] Add support for map for NIO files This unfortunately does not work for most files in JRuby, since we usually use a real native file descriptor on POSIX platforms. This will need to be expanded to work with a native fileno but produce a MappedByteBuffer. --- .../src/main/java/org/jruby/RubyIOBuffer.java | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index da9d1b8e7fd..a538933a178 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -14,11 +14,14 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.ByteList; +import org.jruby.util.io.ChannelFD; import org.jruby.util.io.OpenFile; +import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; import java.util.Arrays; import static org.jruby.RubyBoolean.newBoolean; @@ -109,6 +112,155 @@ public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyOb return buffer; } + @JRubyMethod(name = "map", meta = true) + public static IRubyObject map(ThreadContext context, IRubyObject self, IRubyObject _file) { + RubyFile file = checkFile(context, _file); + + int size = getSizeFromFile(context, file); + + return map(context, file, size, 0, 0); + } + + private static RubyFile checkFile(ThreadContext context, IRubyObject _file) { + RubyIO io = RubyIO.convertToIO(context, _file); + + if (!(io instanceof RubyFile)) { + throw context.runtime.newTypeError(_file, context.runtime.getFile()); + } + + RubyFile file = (RubyFile) io; + return file; + } + + @JRubyMethod(name = "map", meta = true) + public static IRubyObject map(ThreadContext context, IRubyObject self, IRubyObject _file, IRubyObject _size) { + RubyFile file = checkFile(context, _file); + + int size = getSizeForMap(context, file, _size); + + return map(context, file, size, 0, 0); + } + + @JRubyMethod(name = "map", meta = true) + public static IRubyObject map(ThreadContext context, IRubyObject self, IRubyObject _file, IRubyObject _size, IRubyObject _offset) { + RubyFile file = checkFile(context, _file); + + int size = getSizeForMap(context, file, _size); + + // This is the file offset, not the buffer offset: + int offset = RubyNumeric.num2int(_offset); + + return map(context, file, size, offset, 0); + } + + private static int getSizeForMap(ThreadContext context, RubyFile file, IRubyObject _size) { + int size; + if (!_size.isNil()) { + size = extractSize(context, _size); + } else { + size = getSizeFromFile(context, file); + } + return size; + } + + private static int getSizeFromFile(ThreadContext context, RubyFile _file) { + int size; + long file_size = _file.getSize(context); + + // Compiler can confirm that we handled file_size < 0 case: + if (file_size < 0) { + throw context.runtime.newArgumentError("Invalid negative file size!"); + } + // Here, we assume that file_size is positive: + else if (file_size > Integer.MAX_VALUE) { + throw context.runtime.newArgumentError("File larger than address space!"); + } + else { + // This conversion should be safe: + size = (int) file_size; + } + return size; + } + + @JRubyMethod(name = "map", required = 1, optional = 3, meta = true) + public static IRubyObject map(ThreadContext context, IRubyObject self, IRubyObject[] args) { + switch (args.length) { + case 1: + return map(context, self, args[0]); + case 2: + return map(context, self, args[0], args[1]); + case 3: + return map(context, self, args[0], args[1], args[2]); + case 4: + return map(context, self, args[0], args[1], args[2], args[3]); + } + return context.nil; + } + + public static IRubyObject map(ThreadContext context, IRubyObject self, IRubyObject _file, IRubyObject _size, IRubyObject _offset, IRubyObject _flags) { + RubyFile file = checkFile(context, _file); + + int size = getSizeForMap(context, file, _size); + + // This is the file offset, not the buffer offset: + int offset = RubyNumeric.num2int(_offset); + + int flags = RubyNumeric.num2int(_flags); + + return map(context, file, size, offset, flags); + } + + private static RubyIOBuffer map(ThreadContext context, RubyFile file, int size, int offset, int flags) { + RubyIOBuffer buffer = new RubyIOBuffer(context.runtime, context.runtime.getIOBuffer()); + + ChannelFD descriptor = file.getOpenFileChecked().fd(); + + mapFile(context, buffer, descriptor, size, offset, flags); + + return buffer; + } + + private static void mapFile(ThreadContext context, RubyIOBuffer buffer, ChannelFD descriptor, int size, int offset, int flags) { + FileChannel.MapMode protect = FileChannel.MapMode.READ_ONLY; + int access = 0; + + if ((flags & READONLY) == READONLY) { + buffer.flags |= READONLY; + } else { + protect = FileChannel.MapMode.READ_WRITE; + } + + if ((flags & PRIVATE) == PRIVATE) { + buffer.flags |= PRIVATE; + protect = FileChannel.MapMode.PRIVATE; + } else { + // This buffer refers to external buffer. + buffer.flags |= EXTERNAL; + buffer.flags |= SHARED; + } + + ByteBuffer base; + + if (descriptor.chFile == null) { + throw context.runtime.newTypeError("Cannot map non-file resource: " + descriptor.ch); + } + + try { + base = descriptor.chFile.map(protect, offset, size); + } catch (IOException ioe) { + throw Helpers.newIOErrorFromException(context.runtime, ioe); + } + + buffer.base = base; + buffer.size = size; + + buffer.flags |= MAPPED; + } + + public static IRubyObject map(ThreadContext context, IRubyObject self, RubyFile _file, int _size, int _offset, int _flags) { + return context.nil; + } + @JRubyMethod(name = "initialize") public IRubyObject initialize(ThreadContext context) { return initialize(context, DEFAULT_SIZE); From d5742f3f96dba8801ad9a7065b65289085649dcb Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:25:55 -0500 Subject: [PATCH 27/56] Never allow null buffer through --- core/src/main/java/org/jruby/RubyIOBuffer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index a538933a178..149b4166209 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -69,6 +69,8 @@ public static RubyClass createIOBufferClass(Ruby runtime) { public static final int NETWORK_ENDIAN = BIG_ENDIAN; public static RubyIOBuffer newBuffer(Ruby runtime, ByteBuffer base, int size, int flags) { + if (base == null) return newBuffer(runtime, size, flags); + return new RubyIOBuffer(runtime, runtime.getIOBuffer(), base, size, flags); } From 38f8458929b5cc0e75fb3e2c25284cf1c3b22684 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:33:35 -0500 Subject: [PATCH 28/56] Mapped string should be READONLY --- core/src/main/java/org/jruby/RubyIOBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 149b4166209..205591d257e 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -105,7 +105,7 @@ public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyOb ByteList bytes = string.getByteList(); int size = bytes.realSize(); - RubyIOBuffer buffer = newBuffer(context.runtime, ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size), size, flagsForSize(size)); + RubyIOBuffer buffer = newBuffer(context.runtime, ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size), size, READONLY); if (isGiven) { return block.yieldSpecific(context, buffer); From 737b71abfe2edefc82a6518b8ee49722ec0a9e04 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 20:53:13 -0500 Subject: [PATCH 29/56] Update for logic This doesn't pass test_string_mapped because we do not lock String objects like MRI does and cannot raise an error when the String owned by the Buffer is modified independently. --- core/src/main/java/org/jruby/RubyIOBuffer.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 205591d257e..504bd18818f 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -93,21 +93,27 @@ public RubyIOBuffer(Ruby runtime, RubyClass metaClass, ByteBuffer base, int size @JRubyMethod(name = "for", meta = true) public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyObject _string, Block block) { RubyString string = _string.convertToString(); + int flags = string.isFrozen() ? READONLY : 0; // If the string is frozen, both code paths are okay. // If the string is not frozen, if a block is not given, it must be frozen. - boolean isGiven = block.isGiven(); - if (!isGiven) { + if (!block.isGiven()) { // This internally returns the source string if it's already frozen. string = string.newFrozen(); + flags = READONLY; + } else { + if ((flags & READONLY) != READONLY) { + string.modify(); + } } ByteList bytes = string.getByteList(); int size = bytes.realSize(); + ByteBuffer wrap = ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size); - RubyIOBuffer buffer = newBuffer(context.runtime, ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size), size, READONLY); + RubyIOBuffer buffer = newBuffer(context.runtime, wrap, size, flags); - if (isGiven) { + if (block.isGiven()) { return block.yieldSpecific(context, buffer); } From c726bcff6cdb3627757f6dabf4896e37a6042212 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 6 Oct 2023 21:04:48 -0500 Subject: [PATCH 30/56] Limit incoming buffer to avoid overflow --- core/src/main/java/org/jruby/RubyIOBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 504bd18818f..2f47166971a 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -735,8 +735,8 @@ public void resize(ThreadContext context, int size) { // no special behavior for isInternal=true since we do not control the internals of ByteBuffers. ByteBuffer newBase = this.base.isDirect() ? ByteBuffer.allocateDirect(size) : ByteBuffer.allocate(size); + this.base.limit(Math.min(size, this.base.capacity())); newBase.put(this.base); - this.base.clear(); newBase.clear(); this.base = newBase; From ba0d4d71c25193df8a0c4b450ee07d0e107ef117 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 19 Oct 2023 20:05:28 -0500 Subject: [PATCH 31/56] Hook up Fiber#blocking to RubyThread field --- core/src/main/java/org/jruby/RubyThread.java | 10 ++++++- .../java/org/jruby/ext/fiber/ThreadFiber.java | 30 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyThread.java b/core/src/main/java/org/jruby/RubyThread.java index 7f0bf5c421d..85dce4543df 100644 --- a/core/src/main/java/org/jruby/RubyThread.java +++ b/core/src/main/java/org/jruby/RubyThread.java @@ -218,7 +218,7 @@ public enum Status { private volatile RubyThread fiberCurrentThread; private IRubyObject scheduler; - private boolean blocking = true; + private boolean blocking = false; private static final AtomicIntegerFieldUpdater INTERRUPT_FLAG_UPDATER = AtomicIntegerFieldUpdater.newUpdater(RubyThread.class, "interruptFlag"); @@ -2626,4 +2626,12 @@ public IRubyObject getSchedulerCurrent() { return getRuntime().getNil(); } + + public boolean isBlocking() { + return blocking; + } + + public void setBlocking(boolean blocking) { + this.blocking = blocking; + } } diff --git a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java index 4993e34b1cc..2341cc55771 100644 --- a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java +++ b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java @@ -70,6 +70,10 @@ private static void nativeThreadLauncher(Ruby runtime, Runnable runnable) { runtime.getFiberExecutor().submit(runnable); } + public boolean isBlocking() { + return thread.isBlocking(); + } + private static class VirtualThreadLauncher implements BiConsumer { @Override public void accept(Ruby ruby, Runnable runnable) { @@ -96,7 +100,7 @@ public static void initRootFiber(ThreadContext context, RubyThread currentThread ThreadFiber rootFiber = new ThreadFiber(runtime, runtime.getFiber(), true); - rootFiber.data = new FiberData(new FiberQueue(runtime), currentThread, rootFiber, false); + rootFiber.data = new FiberData(new FiberQueue(runtime), currentThread, rootFiber); rootFiber.thread = currentThread; context.setRootFiber(rootFiber); } @@ -107,11 +111,11 @@ public IRubyObject initialize(ThreadContext context, Block block) { if (!block.isGiven()) throw runtime.newArgumentError("tried to create Proc object without block"); - data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this, false); + data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this); FiberData currentFiberData = context.getFiber().data; - thread = createThread(runtime, data, currentFiberData.queue, block); + thread = createThread(runtime, data, currentFiberData.queue, block, false); return context.nil; } @@ -134,11 +138,11 @@ public IRubyObject initialize(ThreadContext context, IRubyObject _opts, Block bl } } - data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this, blocking); + data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this); FiberData currentFiberData = context.getFiber().data; - thread = createThread(runtime, data, currentFiberData.queue, block); + thread = createThread(runtime, data, currentFiberData.queue, block, blocking); return context.nil; } @@ -450,7 +454,7 @@ final boolean alive() { return true; } - static RubyThread createThread(final Ruby runtime, final FiberData data, final FiberQueue queue, final Block block) { + static RubyThread createThread(final Ruby runtime, final FiberData data, final FiberQueue queue, final Block block, boolean blocking) { final AtomicReference fiberThread = new AtomicReference(); // retry with GC once @@ -462,8 +466,10 @@ static RubyThread createThread(final Ruby runtime, final FiberData data, final F ThreadContext context = runtime.getCurrentContext(); context.setFiber(data.fiber.get()); context.useRecursionGuardsFrom(data.parent.getContext()); - fiberThread.set(context.getThread()); - context.getThread().setFiberCurrentThread(data.parent); + RubyThread rubyThread = context.getThread(); + fiberThread.set(rubyThread); + rubyThread.setFiberCurrentThread(data.parent); + rubyThread.setBlocking(blocking); Thread thread = Thread.currentThread(); String oldName = thread.getName(); @@ -586,12 +592,12 @@ protected void finalize() throws Throwable { @JRubyMethod(name = "blocking?") public IRubyObject blocking_p(ThreadContext context) { - return RubyBoolean.newBoolean(context, data.blocking); + return RubyBoolean.newBoolean(context, context.getThread().isBlocking()); } @JRubyMethod(name = "blocking?", meta = true) public static IRubyObject blocking_p_s(ThreadContext context, IRubyObject self) { - boolean blocking = context.getFiber().data.blocking; + boolean blocking = context.getThread().isBlocking(); if (!blocking) return context.fals; return RubyFixnum.one(context.runtime); @@ -678,10 +684,9 @@ public RubyThread getThread() { } public static class FiberData { - FiberData(FiberQueue queue, RubyThread parent, ThreadFiber fiber, boolean blocking) { + FiberData(FiberQueue queue, RubyThread parent, ThreadFiber fiber) { this.queue = queue; this.parent = parent; - this.blocking = blocking; this.fiber = new WeakReference(fiber); } @@ -694,7 +699,6 @@ public ThreadFiber getPrev() { final RubyThread parent; final WeakReference fiber; volatile boolean transferred; - final boolean blocking; } volatile FiberData data; From 2da66895602e2cae018d7eb31233355276558005 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 20 Oct 2023 16:22:02 -0500 Subject: [PATCH 32/56] Fixes for scheduler API This cleans up how blocking works and fixes small parts of the scheduler dispatch logic. Fibers start out nonblocking by default, which means entering such a fiber puts the system in a schedulable state (decrements root fiber's default blocking count of 1 to 0). While schedulable, blocking calls will redispatch into the scheduler to give it a chance to handle the blocking operation and optionally transfer to another unblocked fiber. The handling of blocking and unblocking will require some additional work, since currently only a fiber yielding can unblock its parent, but we are getting the pieces wired up. All of CRuby's test/fiber/test_scheduler.rb passes with these changes. --- .../main/java/org/jruby/FiberScheduler.java | 4 +- core/src/main/java/org/jruby/RubyThread.java | 21 +++-- .../java/org/jruby/ext/fiber/ThreadFiber.java | 82 +++++++++++++++---- .../main/java/org/jruby/util/io/OpenFile.java | 4 +- 4 files changed, 86 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index 3a5f0d180f7..b1f2b20e5fa 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -155,10 +155,10 @@ public static IRubyObject close(ThreadContext context, IRubyObject scheduler) { IRubyObject result; result = Helpers.invokeChecked(context, scheduler, "scheduler_close"); - if (result != RubyBasicObject.UNDEF) return result; + if (result != null) return result; result = Helpers.invokeChecked(context, scheduler, "close"); - if (result != RubyBasicObject.UNDEF) return result; + if (result != null) return result; return context.nil; } diff --git a/core/src/main/java/org/jruby/RubyThread.java b/core/src/main/java/org/jruby/RubyThread.java index 85dce4543df..17dc8760637 100644 --- a/core/src/main/java/org/jruby/RubyThread.java +++ b/core/src/main/java/org/jruby/RubyThread.java @@ -218,7 +218,7 @@ public enum Status { private volatile RubyThread fiberCurrentThread; private IRubyObject scheduler; - private boolean blocking = false; + private volatile int blockingCount = 1; private static final AtomicIntegerFieldUpdater INTERRUPT_FLAG_UPDATER = AtomicIntegerFieldUpdater.newUpdater(RubyThread.class, "interruptFlag"); @@ -436,6 +436,11 @@ public void dispose() { // unlock all locked locks unlockAll(); + // close scheduler, if any + if (scheduler != null && !scheduler.isNil()) { + FiberScheduler.close(getContext(), scheduler); + } + // mark thread as DEAD beDead(); } @@ -2620,18 +2625,22 @@ public IRubyObject getScheduler() { // MRI: rb_fiber_scheduler_current_for_threadptr, rb_fiber_scheduler_current public IRubyObject getSchedulerCurrent() { - if (!blocking) { + if (!isBlocking()) { return scheduler; } return getRuntime().getNil(); } - public boolean isBlocking() { - return blocking; + public void incrementBlocking() { + blockingCount++; } - public void setBlocking(boolean blocking) { - this.blocking = blocking; + public void decrementBlocking() { + blockingCount--; + } + + public boolean isBlocking() { + return blockingCount > 0; } } diff --git a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java index 2341cc55771..f29be1be42a 100644 --- a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java +++ b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java @@ -71,7 +71,7 @@ private static void nativeThreadLauncher(Ruby runtime, Runnable runnable) { } public boolean isBlocking() { - return thread.isBlocking(); + return data.blocking; } private static class VirtualThreadLauncher implements BiConsumer { @@ -100,7 +100,7 @@ public static void initRootFiber(ThreadContext context, RubyThread currentThread ThreadFiber rootFiber = new ThreadFiber(runtime, runtime.getFiber(), true); - rootFiber.data = new FiberData(new FiberQueue(runtime), currentThread, rootFiber); + rootFiber.data = new FiberData(new FiberQueue(runtime), currentThread, rootFiber, true); rootFiber.thread = currentThread; context.setRootFiber(rootFiber); } @@ -111,11 +111,11 @@ public IRubyObject initialize(ThreadContext context, Block block) { if (!block.isGiven()) throw runtime.newArgumentError("tried to create Proc object without block"); - data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this); + data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this, false); FiberData currentFiberData = context.getFiber().data; - thread = createThread(runtime, data, currentFiberData.queue, block, false); + thread = createThread(runtime, data, currentFiberData.queue, block); return context.nil; } @@ -131,18 +131,23 @@ public IRubyObject initialize(ThreadContext context, IRubyObject _opts, Block bl boolean blocking = false; if (!opts.isNil()) { - IRubyObject blockingOpt = ArgsUtil.extractKeywordArg(context, opts, "blocking"); + IRubyObject[] blockingPoolOpt = ArgsUtil.extractKeywordArgs(context, opts, "blocking", "pool"); - if (!blockingOpt.isNil()) { - blocking = blockingOpt.isTrue(); + if (blockingPoolOpt != null) { + IRubyObject blockingOpt = blockingPoolOpt[0]; + if (blockingOpt != null && !blockingOpt.isNil()) { + blocking = blockingOpt.isTrue(); + } + + // TODO: pooling } } - data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this); + data = new FiberData(new FiberQueue(runtime), context.getFiberCurrentThread(), this, blocking); FiberData currentFiberData = context.getFiber().data; - thread = createThread(runtime, data, currentFiberData.queue, block, blocking); + thread = createThread(runtime, data, currentFiberData.queue, block); return context.nil; } @@ -185,6 +190,10 @@ public IRubyObject resume(ThreadContext context, IRubyObject[] values) { data.prev = null; } + if (data.blocking) { + context.getFiberCurrentThread().decrementBlocking(); + } + if (result.type == RequestType.RAISE) { throw ((RubyException) result.data).toThrowable(); } @@ -225,12 +234,28 @@ private static FiberRequest exchangeWithFiber(ThreadContext context, FiberData c // interrupted again and must abandon the fiber. try { + adjustThreadBlocking(context, currentFiberData, targetFiberData); + return currentFiberData.queue.pop(context); } catch (RaiseException re) { handleExceptionDuringExchange(context, currentFiberData, targetFiberData, re); // if we get here, we forwarded exception so try once more return currentFiberData.queue.pop(context); + } finally { + adjustThreadBlocking(context, targetFiberData, currentFiberData); + } + } + + private static void adjustThreadBlocking(ThreadContext context, FiberData currentFiberData, FiberData targetFiberData) { + // if fiber we are leaving is blocking, decrement thread blocking count + if (currentFiberData.blocking) { + context.getFiberCurrentThread().decrementBlocking(); + } + + // if fiber we are entering is blocking, increment thread blocking count + if (targetFiberData.blocking) { + context.getFiberCurrentThread().incrementBlocking(); } } @@ -454,7 +479,7 @@ final boolean alive() { return true; } - static RubyThread createThread(final Ruby runtime, final FiberData data, final FiberQueue queue, final Block block, boolean blocking) { + static RubyThread createThread(final Ruby runtime, final FiberData data, final FiberQueue queue, final Block block) { final AtomicReference fiberThread = new AtomicReference(); // retry with GC once @@ -469,7 +494,6 @@ static RubyThread createThread(final Ruby runtime, final FiberData data, final F RubyThread rubyThread = context.getThread(); fiberThread.set(rubyThread); rubyThread.setFiberCurrentThread(data.parent); - rubyThread.setBlocking(blocking); Thread thread = Thread.currentThread(); String oldName = thread.getName(); @@ -592,17 +616,43 @@ protected void finalize() throws Throwable { @JRubyMethod(name = "blocking?") public IRubyObject blocking_p(ThreadContext context) { - return RubyBoolean.newBoolean(context, context.getThread().isBlocking()); + return RubyBoolean.newBoolean(context, isBlocking()); } @JRubyMethod(name = "blocking?", meta = true) public static IRubyObject blocking_p_s(ThreadContext context, IRubyObject self) { - boolean blocking = context.getThread().isBlocking(); + boolean blocking = context.getFiber().isBlocking(); if (!blocking) return context.fals; return RubyFixnum.one(context.runtime); } + @JRubyMethod(name = "blocking", meta = true) + public static IRubyObject blocking(ThreadContext context, IRubyObject self, Block block) { + ThreadFiber currentFiber = context.getFiber(); + boolean blocking = currentFiber.isBlocking(); + + // If we are already blocking, this is essentially a no-op: + if (currentFiber.isBlocking()) { + return block.yieldSpecific(context, currentFiber); + } + + try { + assert !currentFiber.isBlocking() : "fiber was blocking when it should not have been"; + + currentFiber.data.blocking = true; + + // Once the fiber is blocking, and current, we increment the thread blocking state: + context.getFiberCurrentThread().incrementBlocking(); + + return block.yieldSpecific(context, currentFiber); + } finally { + // We are no longer blocking: + currentFiber.data.blocking = false; + context.getFiberCurrentThread().decrementBlocking(); + } + } + @JRubyMethod(name = "backtrace") public IRubyObject backtrace(ThreadContext context) { return backtrace(context, null, null); @@ -644,7 +694,7 @@ public IRubyObject backtrace_locations(ThreadContext context, IRubyObject level, // MRI: rb_fiber_s_schedule_kw and rb_fiber_s_schedule, kw passes on context @JRubyMethod(name = "schedule", meta = true, rest = true, keywords = true) public static IRubyObject schedule(ThreadContext context, IRubyObject self, IRubyObject[] args, Block block) { - RubyThread thread = context.getFiberCurrentThread(); + RubyThread thread = context.getThread(); IRubyObject scheduler = thread.getScheduler(); IRubyObject fiber = context.nil; @@ -684,10 +734,11 @@ public RubyThread getThread() { } public static class FiberData { - FiberData(FiberQueue queue, RubyThread parent, ThreadFiber fiber) { + FiberData(FiberQueue queue, RubyThread parent, ThreadFiber fiber, boolean blocking) { this.queue = queue; this.parent = parent; this.fiber = new WeakReference(fiber); + this.blocking = blocking; } public ThreadFiber getPrev() { @@ -699,6 +750,7 @@ public ThreadFiber getPrev() { final RubyThread parent; final WeakReference fiber; volatile boolean transferred; + volatile boolean blocking; } volatile FiberData data; diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index 600f6cba46d..ea9bb91e77d 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -1479,7 +1479,7 @@ public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD f if (!scheduler.isNil()) { IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); - if (result != RubyBasicObject.UNDEF) { + if (result != null) { FiberScheduler.resultApply(context, result); } } @@ -2455,7 +2455,7 @@ public static int writeInternal(ThreadContext context, OpenFile fptr, ByteBuffer if (!scheduler.isNil()) { IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); - if (result != RubyBasicObject.UNDEF) { + if (result != null) { FiberScheduler.resultApply(context, result); } } From 5fa2ff76b2dd9e4e4180005a989a8bba2a324d34 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 20 Oct 2023 16:32:24 -0500 Subject: [PATCH 33/56] Update test_scheduler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It still all passes 🎉 --- test/mri/fiber/autoload.rb | 3 + test/mri/fiber/scheduler.rb | 225 +++++++++++++++++++++++-------- test/mri/fiber/test_scheduler.rb | 78 +++++++++++ 3 files changed, 248 insertions(+), 58 deletions(-) create mode 100644 test/mri/fiber/autoload.rb diff --git a/test/mri/fiber/autoload.rb b/test/mri/fiber/autoload.rb new file mode 100644 index 00000000000..dcb27164a70 --- /dev/null +++ b/test/mri/fiber/autoload.rb @@ -0,0 +1,3 @@ +sleep 0.01 +module TestFiberSchedulerAutoload +end diff --git a/test/mri/fiber/scheduler.rb b/test/mri/fiber/scheduler.rb index 4138015e4b6..5090271db15 100644 --- a/test/mri/fiber/scheduler.rb +++ b/test/mri/fiber/scheduler.rb @@ -1,8 +1,13 @@ # frozen_string_literal: true # This is an example and simplified scheduler for test purposes. -# It is not efficient for a large number of file descriptors as it uses IO.select(). -# Production Fiber schedulers should use epoll/kqueue/etc. +# - It is not efficient for a large number of file descriptors as it uses +# IO.select(). +# - It does not correctly handle multiple calls to `wait` with the same file +# descriptor and overlapping events. +# - Production fiber schedulers should use epoll/kqueue/etc. Consider using the +# [`io-event`](https://github.com/socketry/io-event) gem instead of this +# scheduler if you want something simple to build on. require 'fiber' require 'socket' @@ -30,7 +35,7 @@ def initialize @closed = false @lock = Thread::Mutex.new - @blocking = 0 + @blocking = Hash.new.compare_by_identity @ready = [] @urgent = IO.pipe @@ -57,8 +62,8 @@ def next_timeout def run # $stderr.puts [__method__, Fiber.current].inspect - while @readable.any? or @writable.any? or @waiting.any? or @blocking.positive? - # Can only handle file descriptors up to 1024... + while @readable.any? or @writable.any? or @waiting.any? or @blocking.any? + # May only handle file descriptors up to 1024... readable, writable = IO.select(@readable.keys + [@urgent.first], @writable.keys, [], next_timeout) # puts "readable: #{readable}" if readable&.any? @@ -115,10 +120,16 @@ def run end end + # A fiber scheduler hook, invoked when the scheduler goes out of scope. def scheduler_close close(true) end + # If the `scheduler_close` hook does not exist, this method `close` will be + # invoked instead when the fiber scheduler goes out of scope. This is legacy + # behaviour, you should almost certainly use `scheduler_close`. The reason for + # this, is `scheduler_close` is called when the scheduler goes out of scope, + # while `close` may be called by the user. def close(internal = false) # $stderr.puts [__method__, Fiber.current].inspect @@ -153,6 +164,7 @@ def current_time Process.clock_gettime(Process::CLOCK_MONOTONIC) end + # This hook is invoked by `Timeout.timeout` and related code. def timeout_after(duration, klass, message, &block) fiber = Fiber.current @@ -171,6 +183,7 @@ def timeout_after(duration, klass, message, &block) end end + # This hook is invoked by `Process.wait`, `system`, and backticks. def process_wait(pid, flags) # $stderr.puts [__method__, pid, flags, Fiber.current].inspect @@ -180,24 +193,47 @@ def process_wait(pid, flags) end.value end + # This hook is invoked by `IO#read` and `IO#write` in the case that `io_read` + # and `io_write` hooks are not available. This implementation is not + # completely general, in the sense that calling `io_wait` multiple times with + # the same `io` and `events` will not work, which is okay for tests but not + # for real code. Correct fiber schedulers should not have this limitation. def io_wait(io, events, duration) # $stderr.puts [__method__, io, events, duration, Fiber.current].inspect + fiber = Fiber.current + unless (events & IO::READABLE).zero? - @readable[io] = Fiber.current + @readable[io] = fiber + readable = true end unless (events & IO::WRITABLE).zero? - @writable[io] = Fiber.current + @writable[io] = fiber + writable = true + end + + if duration + @waiting[fiber] = current_time + duration end Fiber.yield ensure - @readable.delete(io) - @writable.delete(io) + @waiting.delete(fiber) if duration + @readable.delete(io) if readable + @writable.delete(io) if writable + end + + # This hook is invoked by `IO.select`. Using a thread ensures that the + # operation does not block the fiber scheduler. + def io_select(...) + # Emulate the operation using a non-blocking thread: + Thread.new do + IO.select(...) + end.value end - # Used for Kernel#sleep and Thread::Mutex#sleep + # This hook is invoked by `Kernel#sleep` and `Thread::Mutex#sleep`. def kernel_sleep(duration = nil) # $stderr.puts [__method__, duration, Fiber.current].inspect @@ -206,32 +242,35 @@ def kernel_sleep(duration = nil) return true end - # Used when blocking on synchronization (Thread::Mutex#lock, - # Thread::Queue#pop, Thread::SizedQueue#push, ...) + # This hook is invoked by blocking options such as `Thread::Mutex#lock`, + # `Thread::Queue#pop` and `Thread::SizedQueue#push`, which are unblocked by + # other threads/fibers. To unblock a blocked fiber, you should call `unblock` + # with the same `blocker` and `fiber` arguments. def block(blocker, timeout = nil) # $stderr.puts [__method__, blocker, timeout].inspect + fiber = Fiber.current + if timeout - @waiting[Fiber.current] = current_time + timeout + @waiting[fiber] = current_time + timeout begin Fiber.yield ensure # Remove from @waiting in the case #unblock was called before the timeout expired: - @waiting.delete(Fiber.current) + @waiting.delete(fiber) end else - @blocking += 1 + @blocking[fiber] = true begin Fiber.yield ensure - @blocking -= 1 + @blocking.delete(fiber) end end end - # Used when synchronization wakes up a previously-blocked fiber - # (Thread::Mutex#unlock, Thread::Queue#push, ...). - # This might be called from another thread. + # This method is invoked from a thread or fiber to unblock a fiber that is + # blocked by `block`. It is expected to be thread safe. def unblock(blocker, fiber) # $stderr.puts [__method__, blocker, fiber].inspect # $stderr.puts blocker.backtrace.inspect @@ -245,6 +284,9 @@ def unblock(blocker, fiber) io.write_nonblock('.') end + # This hook is invoked by `Fiber.schedule`. Strictly speaking, you should use + # it to create scheduled fibers, but it is not required in practice; + # `Fiber.new` is usually sufficient. def fiber(&block) fiber = Fiber.new(blocking: false, &block) @@ -253,6 +295,9 @@ def fiber(&block) return fiber end + # This hook is invoked by `Addrinfo.getaddrinfo`. Using a thread ensures that + # the operation does not block the fiber scheduler, since `getaddrinfo` is + # usually provided by `libc` and is blocking. def address_resolve(hostname) Thread.new do Addrinfo.getaddrinfo(hostname, nil).map(&:ip_address).uniq @@ -260,85 +305,133 @@ def address_resolve(hostname) end end +# This scheduler class implements `io_read` and `io_write` hooks which require +# `IO::Buffer`. class IOBufferScheduler < Scheduler - EAGAIN = Errno::EAGAIN::Errno + EAGAIN = -Errno::EAGAIN::Errno - def io_read(io, buffer, length) - offset = 0 + def io_read(io, buffer, length, offset) + total = 0 + io.nonblock = true while true maximum_size = buffer.size - offset - result = blocking{io.read_nonblock(maximum_size, exception: false)} - - # blocking{pp read: maximum_size, result: result, length: length} + result = blocking{buffer.read(io, maximum_size, offset)} - case result - when :wait_readable + if result > 0 + total += result + offset += result + break if total >= length + elsif result == 0 + break + elsif result == EAGAIN if length > 0 self.io_wait(io, IO::READABLE, nil) else - return -EAGAIN + return result end - when :wait_writable + elsif result < 0 + return result + end + end + + return total + end + + def io_write(io, buffer, length, offset) + total = 0 + io.nonblock = true + + while true + maximum_size = buffer.size - offset + result = blocking{buffer.write(io, maximum_size, offset)} + + if result > 0 + total += result + offset += result + break if total >= length + elsif result == 0 + break + elsif result == EAGAIN if length > 0 self.io_wait(io, IO::WRITABLE, nil) else - return -EAGAIN + return result end - else - break unless result - - buffer.set_string(result, offset) - - size = result.bytesize - offset += size - break if size >= length - length -= size + elsif result < 0 + return result end end - return offset + return total end - def io_write(io, buffer, length) - offset = 0 + def io_pread(io, buffer, from, length, offset) + total = 0 + io.nonblock = true while true maximum_size = buffer.size - offset + result = blocking{buffer.pread(io, from, maximum_size, offset)} - chunk = buffer.get_string(offset, maximum_size) - result = blocking{io.write_nonblock(chunk, exception: false)} - - # blocking{pp write: maximum_size, result: result, length: length} - - case result - when :wait_readable + if result > 0 + total += result + offset += result + from += result + break if total >= length + elsif result == 0 + break + elsif result == EAGAIN if length > 0 self.io_wait(io, IO::READABLE, nil) else - return -EAGAIN + return result end - when :wait_writable + elsif result < 0 + return result + end + end + + return total + end + + def io_pwrite(io, buffer, from, length, offset) + total = 0 + io.nonblock = true + + while true + maximum_size = buffer.size - offset + result = blocking{buffer.pwrite(io, from, maximum_size, offset)} + + if result > 0 + total += result + offset += result + from += result + break if total >= length + elsif result == 0 + break + elsif result == EAGAIN if length > 0 self.io_wait(io, IO::WRITABLE, nil) else - return -EAGAIN + return result end - else - offset += result - break if result >= length - length -= result + elsif result < 0 + return result end end - return offset + return total end def blocking(&block) - Fiber.new(blocking: true, &block).resume + Fiber.blocking(&block) end end +# This scheduler has a broken implementation of `unblock`` in the sense that it +# raises an exception. This is used to test the behavior of the scheduler when +# unblock raises an exception. class BrokenUnblockScheduler < Scheduler def unblock(blocker, fiber) super @@ -347,6 +440,9 @@ def unblock(blocker, fiber) end end +# This scheduler has a broken implementation of `unblock` in the sense that it +# sleeps. This is used to test the behavior of the scheduler when unblock +# messes with the internal thread state in an unexpected way. class SleepingUnblockScheduler < Scheduler # This method is invoked when the thread is exiting. def unblock(blocker, fiber) @@ -356,3 +452,16 @@ def unblock(blocker, fiber) sleep(0.1) end end + +# This scheduler has a broken implementation of `kernel_sleep` in the sense that +# it invokes a blocking sleep which can cause a deadlock in some cases. +class SleepingBlockingScheduler < Scheduler + def kernel_sleep(duration = nil) + # Deliberaly sleep in a blocking state which can trigger a deadlock if the implementation is not correct. + Fiber.blocking{sleep 0.0001} + + self.block(:sleep, duration) + + return true + end +end diff --git a/test/mri/fiber/test_scheduler.rb b/test/mri/fiber/test_scheduler.rb index 1870ab1c33e..34effad8163 100644 --- a/test/mri/fiber/test_scheduler.rb +++ b/test/mri/fiber/test_scheduler.rb @@ -27,6 +27,18 @@ def test_fiber_new_with_options refute f.blocking? end + def test_fiber_blocking + f = Fiber.new(blocking: false) do + fiber = Fiber.current + refute fiber.blocking? + Fiber.blocking do |_fiber| + assert_equal fiber, _fiber + assert fiber.blocking? + end + end + f.resume + end + def test_closed_at_thread_exit scheduler = Scheduler.new @@ -104,4 +116,70 @@ def test_current_scheduler thread.join end + + def test_autoload + 10.times do + Object.autoload(:TestFiberSchedulerAutoload, File.expand_path("autoload.rb", __dir__)) + + thread = Thread.new do + scheduler = Scheduler.new + Fiber.set_scheduler scheduler + + 10.times do + Fiber.schedule do + Object.const_get(:TestFiberSchedulerAutoload) + end + end + end + + thread.join + ensure + $LOADED_FEATURES.delete(File.expand_path("autoload.rb", __dir__)) + Object.send(:remove_const, :TestFiberSchedulerAutoload) + end + end + + def test_deadlock + mutex = Thread::Mutex.new + condition = Thread::ConditionVariable.new + q = 0.0001 + + signaller = Thread.new do + loop do + mutex.synchronize do + condition.signal + end + sleep q + end + end + + i = 0 + + thread = Thread.new do + scheduler = SleepingBlockingScheduler.new + Fiber.set_scheduler scheduler + + Fiber.schedule do + 10.times do + mutex.synchronize do + condition.wait(mutex) + sleep q + i += 1 + end + end + end + end + + # Wait for 10 seconds at most... if it doesn't finish, it's deadlocked. + thread.join(10) + + # If it's deadlocked, it will never finish, so this will be 0. + assert_equal 10, i + ensure + # Make sure the threads are dead... + thread.kill + signaller.kill + thread.join + signaller.join + end end From 082842dbc6ee11c3fbd88a19985fa34879bf1fb4 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 20 Oct 2023 16:34:45 -0500 Subject: [PATCH 34/56] Add MRI test/fiber/test_scheduler to stdlib run --- test/mri.stdlib.index | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mri.stdlib.index b/test/mri.stdlib.index index 4d8f273cf6a..0d17870ca70 100644 --- a/test/mri.stdlib.index +++ b/test/mri.stdlib.index @@ -21,7 +21,7 @@ fiber/test_backtrace.rb #fiber/test_mutex.rb #fiber/test_process.rb fiber/test_ractor.rb -#fiber/test_scheduler.rb +fiber/test_scheduler.rb #fiber/test_sleep.rb #fiber/test_thread.rb #fiber/test_timeout.rb From 24615aeae79d5f630f96a2eecaf7933aa65c8daa Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 20 Oct 2023 17:23:16 -0500 Subject: [PATCH 35/56] Hook up IO scheduler callbacks correctly Working on failures in mri/fiber/test_io_buffer.rb --- .../main/java/org/jruby/FiberScheduler.java | 38 ++++++++++++--- core/src/main/java/org/jruby/RubyIO.java | 6 +-- .../src/main/java/org/jruby/RubyIOBuffer.java | 4 +- .../java/org/jruby/ext/fiber/FiberQueue.java | 4 +- .../main/java/org/jruby/util/io/OpenFile.java | 48 +++++++++++++++---- 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index b1f2b20e5fa..2509cf088fc 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -60,8 +60,9 @@ public static IRubyObject ioSelectv(ThreadContext context, IRubyObject scheduler } // MRI: rb_fiber_scheduler_io_read - public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length) { - return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, context.runtime.newFixnum(length)); + public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { + Ruby runtime = context.runtime; + return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, runtime.newFixnum(length), runtime.newFixnum(offset)); } public static IRubyObject ioRead(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger length, RubyInteger offset) { @@ -78,8 +79,9 @@ public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, } // MRI: rb_fiber_scheduler_io_write - public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length) { - return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, context.runtime.newFixnum(length)); + public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { + Ruby runtime = context.runtime; + return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, runtime.newFixnum(length), runtime.newFixnum(offset)); } public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger length, RubyInteger offset) { @@ -99,7 +101,19 @@ public static IRubyObject ioPWrite(ThreadContext context, IRubyObject scheduler, public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int size, int length) { RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED); - IRubyObject result = ioRead(context, scheduler, io, buffer, length); + IRubyObject result = ioRead(context, scheduler, io, buffer, length, 0); + + buffer.unlock(context); + buffer.free(context); + + return result; + } + + // MRI: rb_fiber_scheduler_io_pread_memory + public static IRubyObject ioPReadMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int from, int size, int length) { + RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED); + + IRubyObject result = ioPRead(context, scheduler, io, buffer, from, length, 0); buffer.unlock(context); buffer.free(context); @@ -111,7 +125,19 @@ public static IRubyObject ioReadMemory(ThreadContext context, IRubyObject schedu public static IRubyObject ioWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int size, int length) { RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); - IRubyObject result = ioWrite(context, scheduler, io, buffer, length); + IRubyObject result = ioWrite(context, scheduler, io, buffer, length, 0); + + buffer.unlock(context); + buffer.free(context); + + return result; + } + + // MRI: p + public static IRubyObject ioPWriteMemory(ThreadContext context, IRubyObject scheduler, IRubyObject io, ByteBuffer base, int from, int size, int length) { + RubyIOBuffer buffer = RubyIOBuffer.newBuffer(context.runtime, base, size, RubyIOBuffer.LOCKED | RubyIOBuffer.READONLY); + + IRubyObject result = ioPWrite(context, scheduler, io, buffer, from, length, 0); buffer.unlock(context); buffer.free(context); diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java index 7c03fd1679c..1fd0e2c8155 100644 --- a/core/src/main/java/org/jruby/RubyIO.java +++ b/core/src/main/java/org/jruby/RubyIO.java @@ -4816,7 +4816,7 @@ public IRubyObject pread(ThreadContext context, IRubyObject _length, IRubyObject int read; ByteBuffer wrap = ByteBuffer.wrap(strByteList.unsafeBytes(), strByteList.begin(), length); - read = OpenFile.preadInternal(context, fd, wrap, from, length); + read = OpenFile.preadInternal(context, fptr, fd, wrap, from, length); string.setReadLength(read); @@ -4851,7 +4851,7 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject str, IRubyObject of int written; - written = OpenFile.pwriteInternal(context, fd, wrap, off, length); + written = OpenFile.pwriteInternal(context, fptr, fd, wrap, off, length); return context.runtime.newFixnum(written); } @@ -5202,7 +5202,7 @@ public final OpenFile MakeOpenFile() { rb_io_fptr_finalize(runtime, openFile); openFile = null; } - openFile = new OpenFile(runtime.getNil()); + openFile = new OpenFile(this, runtime.getNil()); runtime.addInternalFinalizer(openFile); return openFile; } diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 2f47166971a..93cddc756f4 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -1916,7 +1916,7 @@ private static IRubyObject preadInternal(ThreadContext context, RubyIO io, ByteB try { base.position(offset); base.limit(offset + size); - int result = OpenFile.preadInternal(context, fptr.fd(), base, from, size); + int result = OpenFile.preadInternal(context, fptr, fptr.fd(), base, from, size); return FiberScheduler.result(context.runtime, result, fptr.errno()); } finally { base.clear(); @@ -2114,7 +2114,7 @@ private static IRubyObject pwriteInternal(ThreadContext context, RubyIO io, Byte try { base.position(offset); base.limit(offset + size); - int result = OpenFile.pwriteInternal(context, fptr.fd(), base, from, size); + int result = OpenFile.pwriteInternal(context, fptr, fptr.fd(), base, from, size); return FiberScheduler.result(context.runtime, result, fptr.errno()); } finally { base.clear(); diff --git a/core/src/main/java/org/jruby/ext/fiber/FiberQueue.java b/core/src/main/java/org/jruby/ext/fiber/FiberQueue.java index 50b6484d0bc..0a9aa0b2814 100644 --- a/core/src/main/java/org/jruby/ext/fiber/FiberQueue.java +++ b/core/src/main/java/org/jruby/ext/fiber/FiberQueue.java @@ -50,7 +50,7 @@ public FiberQueue(Ruby runtime) { this.queue = new ArrayBlockingQueue<>(1, false); } - final RubyThread.Task takeTask = new RubyThread.Task() { + private static final RubyThread.Task TAKE_TASK = new RubyThread.Task() { @Override public FiberRequest run(ThreadContext context, FiberQueue queue) throws InterruptedException { return queue.getQueueSafe().take(); @@ -89,7 +89,7 @@ public synchronized void checkShutdown() { public FiberRequest pop(ThreadContext context) { try { - return context.getThread().executeTaskBlocking(context, this, takeTask); + return context.getThread().executeTaskBlocking(context, this, TAKE_TASK); } catch (InterruptedException ie) { throw context.runtime.newThreadError("interrupted in FiberQueue.pop"); } diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index ea9bb91e77d..6a3ec3a7cd7 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -54,7 +54,8 @@ public class OpenFile implements Finalizable { // RB_IO_FPTR_NEW, minus fields that Java already initializes the same way - public OpenFile(IRubyObject nil) { + public OpenFile(RubyIO io, IRubyObject nil) { + this.io = io; runtime = nil.getRuntime(); writeconvAsciicompat = null; writeconvPreEcopts = nil; @@ -147,6 +148,7 @@ public static class Buffer { public final Buffer wbuf = new Buffer(), rbuf = new Buffer(), cbuf = new Buffer(); + public final RubyIO io; public RubyIO tiedIOForWriting; private boolean nonblock = false; @@ -1474,15 +1476,19 @@ public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD f } // rb_io_buffer_read_internal - public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer bufBytes, int buf, int count) { + public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer buffer, int buf, int count) { + // try scheduler first IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.io, buffer, buf, count); if (result != null) { - FiberScheduler.resultApply(context, result); + return FiberScheduler.resultApply(context, result); } } + + // proceed to builtin read logic + // if we can do selection and this is not a non-blocking call, do selection /* @@ -1499,13 +1505,24 @@ simple read(2) because EINTR does not damage the descriptor. preRead(context, fptr, fd); try { - return context.getThread().executeReadWrite(context, fptr, bufBytes, buf, count, READ_TASK); + return context.getThread().executeReadWrite(context, fptr, buffer, buf, count, READ_TASK); } catch (InterruptedException ie) { throw context.runtime.newConcurrencyError("IO operation interrupted"); } } - public static int preadInternal(ThreadContext context, ChannelFD fd, ByteBuffer buffer, int from, int length) { + public static int preadInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer buffer, int from, int length) { + // try scheduler first + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPReadMemory(context, scheduler, fptr.io, buffer, from, length, 0); + + if (result != null) { + return FiberScheduler.resultApply(context, result); + } + } + + // proceed to builtin pread logic try { return context.getThread().executeReadWrite(context, fd, buffer, from, length, PREAD_TASK); } catch (InterruptedException ie) { @@ -1513,10 +1530,21 @@ public static int preadInternal(ThreadContext context, ChannelFD fd, ByteBuffer } } - public static int pwriteInternal(ThreadContext context, ChannelFD fd, ByteBuffer bytes, int from, int length) { + public static int pwriteInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer buffer, int from, int length) { + // try scheduler first + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPWriteMemory(context, scheduler, fptr.io, buffer, from, length, 0); + + if (result != null) { + return FiberScheduler.resultApply(context, result); + } + } + + // proceed to builtin pread logic int written; try { - written = context.getThread().executeReadWrite(context, fd, bytes, from, length, PWRITE_TASK); + written = context.getThread().executeReadWrite(context, fd, buffer, from, length, PWRITE_TASK); } catch (InterruptedException ie) { throw context.runtime.newConcurrencyError("IO operation interrupted"); } @@ -2453,10 +2481,10 @@ public static int writeInternal(ThreadContext context, OpenFile fptr, byte[] buf public static int writeInternal(ThreadContext context, OpenFile fptr, ByteBuffer bufBytes, int buf, int count) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.tiedIOForWriting, bufBytes, buf, count); + IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.io, bufBytes, buf, count); if (result != null) { - FiberScheduler.resultApply(context, result); + return FiberScheduler.resultApply(context, result); } } From 243c56a9d56426c2d968c47ea67f206b2fce4471 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 20 Oct 2023 20:30:50 -0500 Subject: [PATCH 36/56] Update test_io_buffer and make fixes * This includes two PRs to improve behavior and testing: * https://github.com/ruby/ruby/pull/8728 * https://github.com/ruby/ruby/pull/8729 --- .../src/main/java/org/jruby/RubyIOBuffer.java | 120 ++++++++++++++---- test/mri/ruby/test_io_buffer.rb | 100 ++++++++++++--- 2 files changed, 177 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 93cddc756f4..092e0333207 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -78,6 +78,13 @@ public static RubyIOBuffer newBuffer(Ruby runtime, int size, int flags) { return new RubyIOBuffer(runtime, runtime.getIOBuffer(), newBufferBase(runtime, size, flags), size, flags); } + public static RubyIOBuffer newBuffer(ThreadContext context, RubyString string, int flags) { + ByteList bytes = string.getByteList(); + int size = bytes.realSize(); + + return newBuffer(context.runtime, ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size), size, flags); + } + public RubyIOBuffer(Ruby runtime, RubyClass metaClass) { super(runtime, metaClass); } @@ -107,11 +114,7 @@ public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyOb } } - ByteList bytes = string.getByteList(); - int size = bytes.realSize(); - ByteBuffer wrap = ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size); - - RubyIOBuffer buffer = newBuffer(context.runtime, wrap, size, flags); + RubyIOBuffer buffer = newBuffer(context, string, flags); if (block.isGiven()) { return block.yieldSpecific(context, buffer); @@ -120,6 +123,23 @@ public static IRubyObject rbFor(ThreadContext context, IRubyObject self, IRubyOb return buffer; } + @JRubyMethod(meta = true) + public static IRubyObject string(ThreadContext context, IRubyObject self, IRubyObject _length, Block block) { + Ruby runtime = context.runtime; + + int size = _length.convertToInteger().getIntValue(); + if (size < 0) throw runtime.newArgumentError("negative string size (or size too big)"); + RubyString string = RubyString.newString(runtime, new byte[size]); + ByteList bytes = string.getByteList(); + ByteBuffer wrap = ByteBuffer.wrap(bytes.unsafeBytes(), bytes.begin(), size); + + RubyIOBuffer buffer = newBuffer(context.runtime, wrap, size, 0); + + block.yieldSpecific(context, buffer); + + return string; + } + @JRubyMethod(name = "map", meta = true) public static IRubyObject map(ThreadContext context, IRubyObject self, IRubyObject _file) { RubyFile file = checkFile(context, _file); @@ -701,7 +721,7 @@ public IRubyObject slice(ThreadContext context, int offset, int length) { // MRI: io_buffer_validate_range private void validateRange(ThreadContext context, int offset, int length) { if (offset + length > size) { - throw context.runtime.newArgumentError("Specified offset+length exceeds data size!"); + throw context.runtime.newArgumentError("Specified offset+length is bigger than the buffer size!"); } } @@ -1427,6 +1447,7 @@ public IRubyObject copy(ThreadContext context, RubyIOBuffer source, int offset, return RubyFixnum.newFixnum(context.runtime, length); } + // MRI: io_buffer_copy_from public IRubyObject copy(ThreadContext context, RubyString source, int offset, int length, int sourceOffset) { if (sourceOffset > length) { throw context.runtime.newArgumentError("The given source offset is bigger than the source itself!"); @@ -1471,23 +1492,23 @@ public IRubyObject get_string(ThreadContext context) { @JRubyMethod(name = "get_string") public IRubyObject get_string(ThreadContext context, IRubyObject _offset) { - int offset = RubyNumeric.num2int(_offset); + int offset = extractOffset(context, _offset); return getString(context, offset, size, ASCIIEncoding.INSTANCE); } @JRubyMethod(name = "get_string") public IRubyObject get_string(ThreadContext context, IRubyObject _offset, IRubyObject _length) { - int offset = RubyNumeric.num2int(_offset); - int length = RubyNumeric.num2int(_length); + int offset = extractOffset(context, _offset); + int length = extractLength(context, _length, offset); return getString(context, offset, length, ASCIIEncoding.INSTANCE); } @JRubyMethod(name = "get_string") public IRubyObject get_string(ThreadContext context, IRubyObject _offset, IRubyObject _length, IRubyObject _encoding) { - int offset = RubyNumeric.num2int(_offset); - int length = RubyNumeric.num2int(_length); + int offset = extractOffset(context, _offset); + int length = extractLength(context, _length, offset); Encoding encoding = context.runtime.getEncodingService().getEncodingFromObject(_encoding); return getString(context, offset, length, encoding); @@ -1520,7 +1541,7 @@ public IRubyObject set_string(ThreadContext context, IRubyObject _string) { @JRubyMethod(name = "set_string") public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyObject _offset) { RubyString string = _string.convertToString(); - int offset = RubyNumeric.num2int(_offset); + int offset = extractOffset(context, _offset); return copy(context, string, offset, string.size(), 0); } @@ -1528,8 +1549,8 @@ public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyO @JRubyMethod(name = "set_string") public IRubyObject set_string(ThreadContext context, IRubyObject _string, IRubyObject _offset, IRubyObject _length) { RubyString string = _string.convertToString(); - int offset = RubyNumeric.num2int(_offset); - int length = RubyNumeric.num2int(_length); + int offset = extractOffset(context, _offset); + int length = extractLength(context, _length, offset); return copy(context, string, offset, length, 0); } @@ -1739,28 +1760,59 @@ private static void bufferNotInPlace(ByteBuffer a, int aSize) { } @JRubyMethod(name = "read") - public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length) { + public IRubyObject read(ThreadContext context, IRubyObject io) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - RubyInteger lengthInteger = length.convertToInteger(); if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioRead(context, scheduler, io, this, lengthInteger, RubyFixnum.zero(context.runtime)); + Ruby runtime = context.runtime; + IRubyObject result = FiberScheduler.ioRead(context, scheduler, io, this, RubyFixnum.newFixnum(runtime, size), RubyFixnum.zero(runtime)); if (result != UNDEF) { return result; } } + return read(context, io, size, 0); + } + + @JRubyMethod(name = "read") + public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject _length) { + if (_length.isNil()) { + return read(context, io); + } + + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + RubyInteger lengthInteger = _length.convertToInteger(); + + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioRead(context, scheduler, io, this, lengthInteger, RubyFixnum.zero(context.runtime)); + + if (result != null) { + return result; + } + } + return read(context, io, lengthInteger.getIntValue(), 0); } @JRubyMethod(name = "read") - public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject length, IRubyObject offset) { + public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject _length, IRubyObject _offset) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - RubyInteger lengthInteger = length.convertToInteger(); - RubyInteger offsetInteger = offset.convertToInteger(); + RubyInteger offsetInteger = _offset.convertToInteger(); + int offset = offsetInteger.getIntValue(); + + int length; + RubyInteger lengthInteger; + if (_length.isNil()) { + length = size - offset; + lengthInteger = null; + } else { + lengthInteger = _length.convertToInteger(); + length = lengthInteger.getIntValue(); + } if (!scheduler.isNil()) { + if (lengthInteger == null) lengthInteger = RubyFixnum.newFixnum(context.runtime, length); IRubyObject result = FiberScheduler.ioRead(context, scheduler, io, this, lengthInteger, offsetInteger); if (result != UNDEF) { @@ -1768,7 +1820,7 @@ public IRubyObject read(ThreadContext context, IRubyObject io, IRubyObject lengt } } - return read(context, io, lengthInteger.getIntValue(), offsetInteger.getIntValue()); + return read(context, io, length, offset); } public IRubyObject read(ThreadContext context, IRubyObject io, int length, int offset) { @@ -1963,6 +2015,22 @@ private static int extractSize(ThreadContext context, IRubyObject _size) { return RubyNumeric.num2int(_size); } + @JRubyMethod(name = "write") + public IRubyObject write(ThreadContext context, IRubyObject io) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + + if (!scheduler.isNil()) { + Ruby runtime = context.runtime; + IRubyObject result = FiberScheduler.ioWrite(context, scheduler, io, this, RubyFixnum.newFixnum(runtime, size), RubyFixnum.zero(runtime)); + + if (result != null) { + return result; + } + } + + return write(context, io, size, 0); + } + @JRubyMethod(name = "write") public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject length) { IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); @@ -1971,7 +2039,7 @@ public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject leng if (!scheduler.isNil()) { IRubyObject result = FiberScheduler.ioWrite(context, scheduler, io, this, lengthInteger, RubyFixnum.zero(context.runtime)); - if (result != UNDEF) { + if (result != null) { return result; } } @@ -1988,7 +2056,7 @@ public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject leng if (!scheduler.isNil()) { IRubyObject result = FiberScheduler.ioWrite(context, scheduler, io, this, lengthInteger, offsetInteger); - if (result != UNDEF) { + if (result != null) { return result; } } @@ -2029,7 +2097,7 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject _fr Ruby runtime = context.runtime; IRubyObject result = FiberScheduler.ioPWrite(context, scheduler, io, this, fromInteger, RubyFixnum.newFixnum(runtime, length), RubyFixnum.zero(runtime)); - if (result != UNDEF) { + if (result != null) { return result; } } @@ -2048,7 +2116,7 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject _fr if (!scheduler.isNil()) { IRubyObject result = FiberScheduler.ioPWrite(context, scheduler, io, this, fromInteger, lengthInteger, RubyFixnum.zero(context.runtime)); - if (result != UNDEF) { + if (result != null) { return result; } } @@ -2086,7 +2154,7 @@ public IRubyObject pwrite(ThreadContext context, IRubyObject io, IRubyObject _fr if (!scheduler.isNil()) { IRubyObject result = FiberScheduler.ioPWrite(context, scheduler, io, this, fromInteger, lengthInteger, offsetInteger); - if (result != UNDEF) { + if (result != null) { return result; } } diff --git a/test/mri/ruby/test_io_buffer.rb b/test/mri/ruby/test_io_buffer.rb index 1b2160ecc74..dea8388d7dc 100644 --- a/test/mri/ruby/test_io_buffer.rb +++ b/test/mri/ruby/test_io_buffer.rb @@ -80,7 +80,7 @@ def test_file_mapped end def test_file_mapped_invalid - assert_raise NoMethodError do + assert_raise TypeError do IO::Buffer.map("foobar") end end @@ -102,11 +102,6 @@ def test_string_mapped_mutable IO::Buffer.for(string) do |buffer| refute buffer.readonly? - # Cannot modify string as it's locked by the buffer: - assert_raise RuntimeError do - string[0] = "h" - end - buffer.set_value(:U8, 0, "h".ord) # Buffer releases it's ownership of the string: @@ -116,6 +111,16 @@ def test_string_mapped_mutable end end + def test_string_mapped_buffer_locked + string = "Hello World" + IO::Buffer.for(string) do |buffer| + # Cannot modify string as it's locked by the buffer: + assert_raise RuntimeError do + string[0] = "h" + end + end + end + def test_non_string not_string = Object.new @@ -124,6 +129,20 @@ def test_non_string end end + def test_string + result = IO::Buffer.string(12) do |buffer| + buffer.set_string("Hello World!") + end + + assert_equal "Hello World!", result + end + + def test_string_negative + assert_raise ArgumentError do + IO::Buffer.string(-1) + end + end + def test_resize_mapped buffer = IO::Buffer.new @@ -142,6 +161,24 @@ def test_resize_preserve assert_equal message, buffer.get_string(0, message.bytesize) end + def test_resize_zero_internal + buffer = IO::Buffer.new(1) + + buffer.resize(0) + assert_equal 0, buffer.size + + buffer.resize(1) + assert_equal 1, buffer.size + end + + def test_resize_zero_external + buffer = IO::Buffer.for('1') + + assert_raise IO::Buffer::AccessError do + buffer.resize(0) + end + end + def test_compare_same_size buffer1 = IO::Buffer.new(1) assert_equal buffer1, buffer1 @@ -219,6 +256,18 @@ def test_get_string chunk = buffer.get_string(0, message.bytesize, Encoding::BINARY) assert_equal Encoding::BINARY, chunk.encoding + + assert_raise_with_message(ArgumentError, /bigger than the buffer size/) do + buffer.get_string(0, 129) + end + + assert_raise_with_message(ArgumentError, /bigger than the buffer size/) do + buffer.get_string(129) + end + + assert_raise_with_message(ArgumentError, /Offset can't be negative/) do + buffer.get_string(-1) + end end # We check that values are correctly round tripped. @@ -329,21 +378,38 @@ def test_invalidation input.close end - def test_read - # This is currently a bug in IO:Buffer [#19084] which affects extended - # strings. On 32 bit machines, the example below becomes extended, so - # we omit this test until the bug is fixed. - omit if RUBY_ENGINE == 'ruby' && GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + def hello_world_tempfile io = Tempfile.new io.write("Hello World") io.seek(0) - buffer = IO::Buffer.new(128) - buffer.read(io, 5) - - assert_equal "Hello", buffer.get_string(0, 5) + yield io ensure - io.close! if io + io&.close! + end + + def test_read + hello_world_tempfile do |io| + buffer = IO::Buffer.new(128) + buffer.read(io) + assert_equal "Hello", buffer.get_string(0, 5) + end + end + + def test_read_with_with_length + hello_world_tempfile do |io| + buffer = IO::Buffer.new(128) + buffer.read(io, 5) + assert_equal "Hello", buffer.get_string(0, 5) + end + end + + def test_read_with_with_offset + hello_world_tempfile do |io| + buffer = IO::Buffer.new(128) + buffer.read(io, nil, 6) + assert_equal "Hello", buffer.get_string(6, 5) + end end def test_write @@ -351,7 +417,7 @@ def test_write buffer = IO::Buffer.new(128) buffer.set_string("Hello") - buffer.write(io, 5) + buffer.write(io) io.seek(0) assert_equal "Hello", io.read(5) From fbce6e273d4e44020f31858a5d803f5a0d8a04ad Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Mon, 23 Oct 2023 08:42:54 -0500 Subject: [PATCH 37/56] Copypasta mistakes and define some constants --- core/src/main/java/org/jruby/FiberScheduler.java | 4 ++-- core/src/main/java/org/jruby/RubyIOBuffer.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/jruby/FiberScheduler.java b/core/src/main/java/org/jruby/FiberScheduler.java index 2509cf088fc..0a4af4b33ba 100644 --- a/core/src/main/java/org/jruby/FiberScheduler.java +++ b/core/src/main/java/org/jruby/FiberScheduler.java @@ -81,11 +81,11 @@ public static IRubyObject ioPRead(ThreadContext context, IRubyObject scheduler, // MRI: rb_fiber_scheduler_io_write public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, int length, int offset) { Ruby runtime = context.runtime; - return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, runtime.newFixnum(length), runtime.newFixnum(offset)); + return Helpers.invokeChecked(context, scheduler, "io_write", io, buffer, runtime.newFixnum(length), runtime.newFixnum(offset)); } public static IRubyObject ioWrite(ThreadContext context, IRubyObject scheduler, IRubyObject io, IRubyObject buffer, RubyInteger length, RubyInteger offset) { - return Helpers.invokeChecked(context, scheduler, "io_read", io, buffer, length, offset); + return Helpers.invokeChecked(context, scheduler, "io_write", io, buffer, length, offset); } // MRI: rb_fiber_scheduler_io_pwrite diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 092e0333207..5d5e13e6667 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -38,6 +38,11 @@ public static RubyClass createIOBufferClass(Ruby runtime) { IOBuffer.defineAnnotatedMethods(RubyIOBuffer.class); IOBuffer.defineAnnotatedConstants(RubyIOBuffer.class); + RubyClass IO = runtime.getIO(); + + IO.setConstant("READABLE", runtime.newFixnum(OpenFile.READABLE)); + IO.setConstant("WRITABLE", runtime.newFixnum(OpenFile.WRITABLE)); + return IOBuffer; } @@ -2067,7 +2072,7 @@ public IRubyObject write(ThreadContext context, IRubyObject io, IRubyObject leng public IRubyObject write(ThreadContext context, IRubyObject io, int length, int offset) { validateRange(context, offset, length); - ByteBuffer buffer = getBufferForWriting(context); + ByteBuffer buffer = getBufferForReading(context); return writeInternal(context, RubyIO.convertToIO(context, io), buffer, offset, length); } From 277980606584473282bbab55de6ddb86dfced978 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Mon, 23 Oct 2023 09:10:58 -0500 Subject: [PATCH 38/56] Fiber ext was moved into core in 3.0 --- core/src/main/java/org/jruby/Ruby.java | 1 + .../main/java/org/jruby/ext/fiber/ThreadFiber.java | 8 ++++---- core/src/main/ruby/jruby/kernel/enumerator.rb | 4 ++-- lib/ruby/stdlib/fiber.rb | 12 ------------ 4 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 lib/ruby/stdlib/fiber.rb diff --git a/core/src/main/java/org/jruby/Ruby.java b/core/src/main/java/org/jruby/Ruby.java index 03dbc4da8dc..f24ecba7807 100644 --- a/core/src/main/java/org/jruby/Ruby.java +++ b/core/src/main/java/org/jruby/Ruby.java @@ -591,6 +591,7 @@ private void initBootLibraries() { loadService.provide("rational.rb"); loadService.provide("complex.rb"); loadService.provide("thread.rb"); + loadService.provide("fiber.rb"); loadService.provide("ruby2_keywords.rb"); // Load preludes diff --git a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java index f29be1be42a..c676c7dd64b 100644 --- a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java +++ b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java @@ -305,7 +305,7 @@ private static void handleExceptionDuringExchange(ThreadContext context, FiberDa } @JRubyMethod(rest = true) - public IRubyObject __transfer__(ThreadContext context, IRubyObject[] values) { + public IRubyObject transfer(ThreadContext context, IRubyObject[] values) { Ruby runtime = context.runtime; final FiberData data = this.data; @@ -455,13 +455,13 @@ private static FiberData verifyCurrentFiber(ThreadContext context, Ruby runtime) return currentFiberData; } - @JRubyMethod - public IRubyObject __alive__(ThreadContext context) { + @JRubyMethod(name = "alive?") + public IRubyObject alive_p(ThreadContext context) { return RubyBoolean.newBoolean(context, alive()); } @JRubyMethod(meta = true) - public static IRubyObject __current__(ThreadContext context, IRubyObject recv) { + public static IRubyObject current(ThreadContext context, IRubyObject recv) { return context.getFiber(); } diff --git a/core/src/main/ruby/jruby/kernel/enumerator.rb b/core/src/main/ruby/jruby/kernel/enumerator.rb index cc428d9650f..e9835feee00 100644 --- a/core/src/main/ruby/jruby/kernel/enumerator.rb +++ b/core/src/main/ruby/jruby/kernel/enumerator.rb @@ -98,7 +98,7 @@ def next? end def next - reset unless @fiber&.__alive__ + reset unless @fiber&.alive? val = @fiber.resume @@ -109,7 +109,7 @@ def next def rewind fiber, @fiber = @fiber, nil - fiber.send(:__finalize__) if fiber&.__alive__ + fiber.send(:__finalize__) if fiber&.alive? @state.done = false end diff --git a/lib/ruby/stdlib/fiber.rb b/lib/ruby/stdlib/fiber.rb deleted file mode 100644 index d568fc8e963..00000000000 --- a/lib/ruby/stdlib/fiber.rb +++ /dev/null @@ -1,12 +0,0 @@ -## -# Extensions to Fiber. In JRuby, these are all still defined in the normal -# Fiber class, so we alias them to the right names here. -# -class Fiber - class << self - alias current __current__ - end - - alias transfer __transfer__ - alias alive? __alive__ -end \ No newline at end of file From 1361c18ca045eab66091a767a46cf7d7fbb90afe Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Wed, 25 Oct 2023 10:20:18 -0500 Subject: [PATCH 39/56] Fix length handling in select!/reject! ensure The logic here had diverged somewhat from CRuby's logic, and did not reacquire the array's begin or length as in CRuby. This led to bad length calculation when there were additional mutations during the select!/reject! loop. Fixes #7976 --- core/src/main/java/org/jruby/RubyArray.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyArray.java b/core/src/main/java/org/jruby/RubyArray.java index 70525ae0936..6bd4ce03b8d 100644 --- a/core/src/main/java/org/jruby/RubyArray.java +++ b/core/src/main/java/org/jruby/RubyArray.java @@ -2924,7 +2924,7 @@ public IRubyObject select_bang(ThreadContext context, Block block) { } finally { if (modified) checkFrozen(); - selectBangEnsure(runtime, len, beg, len0, len1); + selectBangEnsure(runtime, len0, len1); } } @@ -3082,17 +3082,20 @@ public IRubyObject rejectBang(ThreadContext context, Block block) { } finally { if (modified) checkFrozen(); - selectBangEnsure(runtime, realLength, beg, len0, len1); + selectBangEnsure(runtime, len0, len1); } } // MRI: select_bang_ensure - private void selectBangEnsure(final Ruby runtime, final int len, final int beg, - int i1, int i2) { - if (i2 < i1) { - realLength = len - i1 + i2; + private void selectBangEnsure(final Ruby runtime, int i1, int i2) { + int len = realLength; + + if (i2 < len && i2 < i1) { + int tail = 0; + int beg = begin; if (i1 < len) { - safeArrayCopy(runtime, values, beg + i1, values, beg + i2, len - i1); + tail = len - i1; + safeArrayCopy(runtime, values, beg + i1, values, beg + i2, tail); } else if (realLength > 0) { // nil out left-overs to avoid leaks (MRI doesn't) @@ -3102,6 +3105,7 @@ else if (realLength > 0) { throw concurrentModification(runtime, ex); } } + realLength = i2 + tail; } } From 74fa59ea8762892817c8d3ff1e1262661ae5d826 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 26 Oct 2023 00:31:24 -0500 Subject: [PATCH 40/56] Remove unnecessary cast of kwargs The intent here was to pass in what might be kwargs if the keyword flag was set, indicating that kwargs were being passed in. This assumption seems ok so I'm not sure why it doesn't hold. In any case, the cast here is unnecessary and the IRubyObject form will return null for non-RubyHashes. The code that triggered this does deal directly with kwargs, but as a kwrest, forwarding it through a block: https://github.com/rails/rails/blob/v7.1.1/activesupport/lib/active_support/testing/time_helpers.rb#L176-L183 This could indicate JRuby failing to pass along a keyword flag when forwarding kwrest. The additional changes here are: * Fall back on non-kwargs behavior if last arg turns out not to be a hash. * Remove a redundant reset of the `callInfo` that governs keyword- passing. Fixes #7982 --- core/src/main/java/org/jruby/RubyTime.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/jruby/RubyTime.java b/core/src/main/java/org/jruby/RubyTime.java index 5a81b3a78b7..a3c960c29b8 100644 --- a/core/src/main/java/org/jruby/RubyTime.java +++ b/core/src/main/java/org/jruby/RubyTime.java @@ -1715,16 +1715,21 @@ private IRubyObject initializeNow(ThreadContext context, IRubyObject zone) { public IRubyObject initialize(ThreadContext context, IRubyObject[] args) { boolean keywords = hasKeywords(context.resetCallInfo()); IRubyObject zone = null; - context.resetCallInfo(); IRubyObject nil = context.nil; int argc = args.length; if (keywords) { - IRubyObject in = ArgsUtil.extractKeywordArg(context, (RubyHash) args[argc - 1], "in"); - if (in != null && argc > 7) { - throw context.runtime.newArgumentError("timezone argument given as positional and keyword arguments"); + IRubyObject in = ArgsUtil.extractKeywordArg(context, args[argc - 1], "in"); + if (in != null) { + if (argc > 7) { + throw context.runtime.newArgumentError("timezone argument given as positional and keyword arguments"); + } + zone = in; + } else { + if (argc > 6) { + zone = args[6]; + } } - zone = in; } else if (argc > 6) { zone = args[6]; } From c14d982081467660852f2fd90efaea6e950279e5 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 26 Oct 2023 01:05:59 -0500 Subject: [PATCH 41/56] Replace gem path string as-is, no regex Fixes #7983 --- lib/pom.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pom.rb b/lib/pom.rb index c95e17af36a..38f91d860cd 100644 --- a/lib/pom.rb +++ b/lib/pom.rb @@ -339,7 +339,7 @@ def installer.ensure_required_ruby_version_met; end spec.executables.each do |f| bin = Dir.glob(File.join( gems, "#{gem_name}*", spec.bindir ))[0] source = File.join( bin, f ) - target = File.join( bin_stubs, source.sub( /#{gems}/, '' ) ) + target = File.join( bin_stubs, source.sub( gems, '' ) ) log "copy #{f} to #{target}" FileUtils.mkdir_p( File.dirname( target ) ) FileUtils.cp_r( source, target ) From ce2476dc90167e168ff5778f799931f86eee1395 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 26 Oct 2023 13:29:14 -0500 Subject: [PATCH 42/56] Hide FilenoUtil warning behind native.verbose property My goal with this warning was to avoid bug reports about native process functionality not working well when running without the module flags we need to dig up native file descriptors from Java channels. Since I added this, however, many things have changed: * Native process IO does not go through Java channels anymore; it instead uses real native IO all the way through. * File IO now also usually goes through JNR using a native file descriptor. * The fileno extraction is now primarily used to provide a fileno for RubyIO objects, but most Ruby code will not need the real fileno. * Module flags are frequently difficult to inject into a jar-based usage of JRuby, so these flags may not be helpful anyway. I believe a warning about degraded functionality is important, but it should be at point-of-use rather than globally just in case the native fileno functionality might become necessary in the future. As such, I am hiding this global warning behind the native.verbose property, and we can consider adding other more targeted warnings going forward. --- .../main/java/org/jruby/util/io/FilenoUtil.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/jruby/util/io/FilenoUtil.java b/core/src/main/java/org/jruby/util/io/FilenoUtil.java index 22ddad63b83..795002c79a2 100644 --- a/core/src/main/java/org/jruby/util/io/FilenoUtil.java +++ b/core/src/main/java/org/jruby/util/io/FilenoUtil.java @@ -16,6 +16,7 @@ import org.jruby.javasupport.JavaUtil; import org.jruby.platform.Platform; import org.jruby.runtime.Helpers; +import org.jruby.util.cli.Options; import org.jruby.util.collections.NonBlockingHashMapLong; import org.jruby.util.log.Logger; import org.jruby.util.log.LoggerFactory; @@ -269,14 +270,16 @@ private static class ReflectiveAccess { FILE_DESCRIPTOR_SET_FILENO_HANDLE = fdSetFileno; if (selChImplGetFD == null || fileChannelGetFD == null || fdGetFileno == null) { - // Warn users since we don't currently handle half-native process control. - Module module = Modules.getModule(ReflectiveAccess.class); - String moduleName = module.getName(); - if (moduleName == null) { - moduleName = "ALL-UNNAMED"; + if (Options.NATIVE_VERBOSE.load()) { + // Warn users since we don't currently handle half-native process control. + Module module = Modules.getModule(ReflectiveAccess.class); + String moduleName = module.getName(); + if (moduleName == null) { + moduleName = "ALL-UNNAMED"; + } + LOG.warn("Native IO integration requires open access to the JDK IO subsystem\n" + + "Pass '--add-opens java.base/sun.nio.ch=" + moduleName + " --add-opens java.base/java.io=" + moduleName + "' to enable."); } - LOG.warn("Native subprocess control requires open access to the JDK IO subsystem\n" + - "Pass '--add-opens java.base/sun.nio.ch=" + moduleName + " --add-opens java.base/java.io=" + moduleName + "' to enable."); } } From ea354852626be5caa88febbb49dcd4ea34a44c55 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 26 Oct 2023 14:08:27 -0500 Subject: [PATCH 43/56] Move remaining logical chunks out of Bootstrap --- core/src/main/java/org/jruby/RubyModule.java | 6 +- .../java/org/jruby/ir/targets/JVMVisitor.java | 8 +- .../org/jruby/ir/targets/indy/Bootstrap.java | 315 +----------------- .../ir/targets/indy/CallInfoBootstrap.java | 37 ++ .../targets/indy/CallSiteCacheBootstrap.java | 44 +++ .../jruby/ir/targets/indy/CheckpointSite.java | 60 ++++ .../targets/indy/ConstructBlockBootstrap.java | 104 ++++++ .../jruby/ir/targets/indy/CoverageSite.java | 44 +++ .../targets/indy/HeapVariableBootstrap.java | 84 +++++ .../ir/targets/indy/IndyBlockCompiler.java | 2 +- .../targets/indy/IndyCheckpointCompiler.java | 2 +- .../targets/indy/IndyInvocationCompiler.java | 7 +- .../indy/IndyLocalVariableCompiler.java | 4 +- .../ir/targets/indy/IndyValueCompiler.java | 2 +- .../ir/targets/indy/MetaClassBootstrap.java | 57 ++++ .../runtime/invokedynamic/MathLinker.java | 19 +- 16 files changed, 468 insertions(+), 327 deletions(-) create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/CallInfoBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/CallSiteCacheBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/CheckpointSite.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/ConstructBlockBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/CoverageSite.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/HeapVariableBootstrap.java create mode 100644 core/src/main/java/org/jruby/ir/targets/indy/MetaClassBootstrap.java diff --git a/core/src/main/java/org/jruby/RubyModule.java b/core/src/main/java/org/jruby/RubyModule.java index 0e4d06c9ff9..75306f95c77 100644 --- a/core/src/main/java/org/jruby/RubyModule.java +++ b/core/src/main/java/org/jruby/RubyModule.java @@ -5108,6 +5108,10 @@ private RaiseException cannotRemoveError(String id) { return getRuntime().newNameError(str(runtime, "cannot remove ", ids(runtime, id), " for ", types(runtime, this)), id); } + public static boolean testModuleMatch(ThreadContext context, IRubyObject arg0, int id) { + return arg0 instanceof RubyModule && ((RubyModule) arg0).id == id; + } + // ////////////////// INTERNAL MODULE VARIABLE API METHODS //////////////// @@ -6159,5 +6163,5 @@ public synchronized void defineAliases(List aliases, String oldId) { */ private static final MethodHandle testModuleMatch = Binder .from(boolean.class, ThreadContext.class, IRubyObject.class, int.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "testModuleMatch"); + .invokeStaticQuiet(LOOKUP, RubyModule.class, "testModuleMatch"); } diff --git a/core/src/main/java/org/jruby/ir/targets/JVMVisitor.java b/core/src/main/java/org/jruby/ir/targets/JVMVisitor.java index 1f59fffbc3e..2118c6d18bb 100644 --- a/core/src/main/java/org/jruby/ir/targets/JVMVisitor.java +++ b/core/src/main/java/org/jruby/ir/targets/JVMVisitor.java @@ -48,8 +48,9 @@ import org.jruby.ir.representations.BasicBlock; import org.jruby.ir.runtime.IRRuntimeHelpers; import org.jruby.ir.targets.IRBytecodeAdapter.BlockPassType; -import org.jruby.ir.targets.indy.Bootstrap; import org.jruby.ir.targets.indy.CallTraceSite; +import org.jruby.ir.targets.indy.CoverageSite; +import org.jruby.ir.targets.indy.MetaClassBootstrap; import org.jruby.parser.StaticScope; import org.jruby.runtime.ArgumentDescriptor; import org.jruby.runtime.Block; @@ -57,7 +58,6 @@ import org.jruby.runtime.DynamicScope; import org.jruby.runtime.Frame; import org.jruby.runtime.Helpers; -import org.jruby.runtime.RubyEvent; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.Visibility; import org.jruby.runtime.builtin.IRubyObject; @@ -1515,7 +1515,7 @@ public void DefineMetaClassInstr(DefineMetaClassInstr definemetaclassinstr) { jvmAdapter().invokedynamic( "openMetaClass", sig(DynamicMethod.class, ThreadContext.class, IRubyObject.class, String.class, StaticScope.class), - Bootstrap.OPEN_META_CLASS, + MetaClassBootstrap.OPEN_META_CLASS, bodyHandle, scopeHandle, setScopeHandle, @@ -1738,7 +1738,7 @@ public void LineNumberInstr(LineNumberInstr linenumberinstr) { jvmAdapter().invokedynamic( "coverLine", sig(void.class, ThreadContext.class), - Bootstrap.coverLineHandle(), + CoverageSite.COVER_LINE_BOOTSTRAP, jvm.methodData().scope.getFile(), linenumberinstr.getLineNumber(), linenumberinstr.oneshot ? 1 : 0); diff --git a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java index 1aac20c8cd8..2c20d13e35d 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/Bootstrap.java @@ -26,47 +26,19 @@ package org.jruby.ir.targets.indy; -import com.headius.invokebinder.Binder; -import org.jruby.Ruby; -import org.jruby.RubyModule; import org.jruby.internal.runtime.methods.DynamicMethod; -import org.jruby.ir.JIT; -import org.jruby.ir.runtime.IRRuntimeHelpers; -import org.jruby.parser.StaticScope; -import org.jruby.runtime.Binding; import org.jruby.runtime.Block; -import org.jruby.runtime.CallType; -import org.jruby.runtime.CompiledIRBlockBody; -import org.jruby.runtime.DynamicScope; -import org.jruby.runtime.Frame; -import org.jruby.runtime.Helpers; -import org.jruby.runtime.ThreadContext; -import org.jruby.runtime.builtin.IRubyObject; -import org.jruby.runtime.callsite.CachingCallSite; -import org.jruby.runtime.callsite.FunctionalCachingCallSite; -import org.jruby.runtime.callsite.MonomorphicCallSite; -import org.jruby.runtime.callsite.VariableCachingCallSite; -import org.jruby.runtime.invokedynamic.MathLinker; -import org.jruby.runtime.opto.Invalidator; -import org.jruby.runtime.scope.DynamicScopeGenerator; import org.jruby.util.log.Logger; import org.jruby.util.log.LoggerFactory; import org.objectweb.asm.Handle; import org.objectweb.asm.Opcodes; import java.lang.invoke.CallSite; -import java.lang.invoke.ConstantCallSite; -import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; -import java.lang.invoke.MutableCallSite; -import java.lang.invoke.SwitchPoint; import static java.lang.invoke.MethodHandles.Lookup; -import static java.lang.invoke.MethodHandles.constant; import static java.lang.invoke.MethodHandles.dropArguments; -import static java.lang.invoke.MethodHandles.insertArguments; -import static java.lang.invoke.MethodHandles.lookup; import static java.lang.invoke.MethodType.methodType; import static org.jruby.runtime.Helpers.arrayOf; import static org.jruby.util.CodegenUtils.p; @@ -81,95 +53,12 @@ public class Bootstrap { private static final Logger LOG = LoggerFactory.getLogger(Bootstrap.class); private static final Lookup LOOKUP = MethodHandles.lookup(); - public static final Handle CALLSITE = new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "callSite", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, int.class), - false); - - public static CallSite callSite(Lookup lookup, String name, MethodType type, String id, int callType) { - return new ConstantCallSite(constant(CachingCallSite.class, callSite(id, callType))); - } - - private static CachingCallSite callSite(String id, int callType) { - switch (CallType.fromOrdinal(callType)) { - case NORMAL: - return new MonomorphicCallSite(id); - case FUNCTIONAL: - return new FunctionalCachingCallSite(id); - case VARIABLE: - return new VariableCachingCallSite(id); - default: - throw new RuntimeException("BUG: Unexpected call type " + callType + " in JVM6 invoke logic"); - } - } - - public static final Handle OPEN_META_CLASS = new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "openMetaClass", - sig(CallSite.class, Lookup.class, String.class, MethodType.class, MethodHandle.class, MethodHandle.class, MethodHandle.class, int.class, int.class, int.class), - false); - - @JIT - public static CallSite openMetaClass(Lookup lookup, String name, MethodType type, MethodHandle body, MethodHandle scope, MethodHandle setScope, int line, int dynscopeEliminated, int refinements) { - try { - StaticScope staticScope = (StaticScope) scope.invokeExact(); - return new ConstantCallSite(insertArguments(OPEN_META_CLASS_HANDLE, 4, body, staticScope, setScope, line, dynscopeEliminated == 1 ? true : false, refinements == 1 ? true : false)); - } catch (Throwable t) { - Helpers.throwException(t); - return null; - } - } - - private static final MethodHandle OPEN_META_CLASS_HANDLE = - Binder - .from(DynamicMethod.class, ThreadContext.class, IRubyObject.class, String.class, StaticScope.class, MethodHandle.class, StaticScope.class, MethodHandle.class, int.class, boolean.class, boolean.class) - .invokeStaticQuiet(LOOKUP, Bootstrap.class, "openMetaClass"); - - @JIT - public static DynamicMethod openMetaClass(ThreadContext context, IRubyObject object, String descriptor, StaticScope parent, MethodHandle body, StaticScope scope, MethodHandle setScope, int line, boolean dynscopeEliminated, boolean refinements) throws Throwable { - if (scope == null) { - scope = Helpers.restoreScope(descriptor, parent); - setScope.invokeExact(scope); - } - return IRRuntimeHelpers.newCompiledMetaClass(context, body, scope, object, line, dynscopeEliminated, refinements); - } - - /////////////////////////////////////////////////////////////////////////// - // Fixnum binding - - public static boolean testModuleMatch(ThreadContext context, IRubyObject arg0, int id) { - return arg0 instanceof RubyModule && ((RubyModule)arg0).id == id; - } - - public static Handle getFixnumOperatorHandle() { - return getBootstrapHandle("fixnumOperatorBootstrap", MathLinker.class, BOOTSTRAP_LONG_STRING_INT_SIG); - } - - public static Handle getFloatOperatorHandle() { - return getBootstrapHandle("floatOperatorBootstrap", MathLinker.class, BOOTSTRAP_DOUBLE_STRING_INT_SIG); - } - - public static Handle checkpointHandle() { - return getBootstrapHandle("checkpointBootstrap", BOOTSTRAP_BARE_SIG); - } - - public static Handle callInfoHandle() { - return getBootstrapHandle("callInfoBootstrap", BOOTSTRAP_INT_SIG); - } - - public static Handle coverLineHandle() { - return getBootstrapHandle("coverLineBootstrap", sig(CallSite.class, Lookup.class, String.class, MethodType.class, String.class, int.class, int.class)); - } - - public static Handle getHeapLocalHandle() { - return getBootstrapHandle("getHeapLocalBootstrap", BOOTSTRAP_INT_INT_SIG); + static String logMethod(DynamicMethod method) { + return "[#" + method.getSerialNumber() + " " + method.getImplementationClass().getMethodLocation() + "]"; } - public static Handle getHeapLocalOrNilHandle() { - return getBootstrapHandle("getHeapLocalOrNilBootstrap", BOOTSTRAP_INT_INT_SIG); + static String logBlock(Block block) { + return "[" + block.getBody().getFile() + ":" + block.getBody().getLine() + "]"; } public static Handle getBootstrapHandle(String name, String sig) { @@ -184,200 +73,4 @@ public static Handle getBootstrapHandle(String name, Class type, String sig) { sig, false); } - - public static CallSite checkpointBootstrap(Lookup lookup, String name, MethodType type) throws Throwable { - MutableCallSite site = new MutableCallSite(type); - MethodHandle handle = lookup.findStatic(Bootstrap.class, "checkpointFallback", methodType(void.class, MutableCallSite.class, ThreadContext.class)); - - handle = handle.bindTo(site); - site.setTarget(handle); - - return site; - } - - public static void checkpointFallback(MutableCallSite site, ThreadContext context) throws Throwable { - Ruby runtime = context.runtime; - Invalidator invalidator = runtime.getCheckpointInvalidator(); - - MethodHandle target = Binder - .from(void.class, ThreadContext.class) - .nop(); - MethodHandle fallback = lookup().findStatic(Bootstrap.class, "checkpointFallback", methodType(void.class, MutableCallSite.class, ThreadContext.class)); - fallback = fallback.bindTo(site); - - target = ((SwitchPoint)invalidator.getData()).guardWithTest(target, fallback); - - site.setTarget(target); - - // poll for events once since we've ended up back in fallback - context.pollThreadEvents(); - } - - public static CallSite callInfoBootstrap(Lookup lookup, String name, MethodType type, int callInfo) throws Throwable { - MethodHandle handle; - if (callInfo == 0) { - handle = lookup.findVirtual(ThreadContext.class, "clearCallInfo", methodType(void.class)); - } else { - handle = lookup.findStatic(IRRuntimeHelpers.class, "setCallInfo", methodType(void.class, ThreadContext.class, int.class)); - handle = insertArguments(handle, 1, callInfo); - } - - return new ConstantCallSite(handle); - } - - public static CallSite coverLineBootstrap(Lookup lookup, String name, MethodType type, String filename, int line, int oneshot) throws Throwable { - MutableCallSite site = new MutableCallSite(type); - MethodHandle handle = lookup.findStatic(Bootstrap.class, "coverLineFallback", methodType(void.class, MutableCallSite.class, ThreadContext.class, String.class, int.class, boolean.class)); - - handle = handle.bindTo(site); - handle = insertArguments(handle, 1, filename, line, oneshot != 0); - site.setTarget(handle); - - return site; - } - - public static void coverLineFallback(MutableCallSite site, ThreadContext context, String filename, int line, boolean oneshot) throws Throwable { - IRRuntimeHelpers.updateCoverage(context, filename, line); - - if (oneshot) site.setTarget(Binder.from(void.class, ThreadContext.class).dropAll().nop()); - } - - public static CallSite getHeapLocalBootstrap(Lookup lookup, String name, MethodType type, int depth, int location) throws Throwable { - // no null checking needed for method bodies - MethodHandle getter; - Binder binder = Binder - .from(type); - - if (depth == 0) { - if (location < DynamicScopeGenerator.SPECIALIZED_GETS.size()) { - getter = binder.invokeVirtualQuiet(LOOKUP, DynamicScopeGenerator.SPECIALIZED_GETS.get(location)); - } else { - getter = binder - .insert(1, location) - .invokeVirtualQuiet(LOOKUP, "getValueDepthZero"); - } - } else { - getter = binder - .insert(1, arrayOf(int.class, int.class), location, depth) - .invokeVirtualQuiet(LOOKUP, "getValue"); - } - - ConstantCallSite site = new ConstantCallSite(getter); - - return site; - } - - public static CallSite getHeapLocalOrNilBootstrap(Lookup lookup, String name, MethodType type, int depth, int location) throws Throwable { - MethodHandle getter; - Binder binder = Binder - .from(type) - .filter(1, LiteralValueBootstrap.contextValue(lookup, "nil", methodType(IRubyObject.class, ThreadContext.class)).dynamicInvoker()); - - if (depth == 0) { - if (location < DynamicScopeGenerator.SPECIALIZED_GETS_OR_NIL.size()) { - getter = binder.invokeVirtualQuiet(LOOKUP, DynamicScopeGenerator.SPECIALIZED_GETS_OR_NIL.get(location)); - } else { - getter = binder - .insert(1, location) - .invokeVirtualQuiet(LOOKUP, "getValueDepthZeroOrNil"); - } - } else { - getter = binder - .insert(1, arrayOf(int.class, int.class), location, depth) - .invokeVirtualQuiet(LOOKUP, "getValueOrNil"); - } - - ConstantCallSite site = new ConstantCallSite(getter); - - return site; - } - - public static Handle prepareBlock() { - return new Handle( - Opcodes.H_INVOKESTATIC, - p(Bootstrap.class), - "prepareBlock", - sig(CallSite.class, Lookup.class, String.class, - MethodType.class, MethodHandle.class, MethodHandle.class, MethodHandle.class, MethodHandle.class, String.class, long.class, - String.class, int.class, String.class), - false); - } - - public static CallSite prepareBlock(Lookup lookup, String name, MethodType type, - MethodHandle bodyHandle, MethodHandle scopeHandle, MethodHandle setScopeHandle, MethodHandle parentHandle, String scopeDescriptor, long encodedSignature, - String file, int line, String encodedArgumentDescriptors - ) throws Throwable { - StaticScope staticScope = (StaticScope) scopeHandle.invokeExact(); - - if (staticScope == null) { - staticScope = Helpers.restoreScope(scopeDescriptor, (StaticScope) parentHandle.invokeExact()); - setScopeHandle.invokeExact(staticScope); - } - - CompiledIRBlockBody body = new CompiledIRBlockBody(bodyHandle, staticScope, file, line, encodedArgumentDescriptors, encodedSignature); - - Binder binder = Binder.from(type); - - binder = binder.fold(FRAME_SCOPE_BINDING); - - // This optimization can't happen until we can see into the method we're calling to know if it reifies the block - if (false) { - /* - if (needsBinding) { - if (needsFrame) { - FullInterpreterContext fic = scope.getExecutionContext(); - if (fic.needsBinding()) { - if (fic.needsFrame()) { - binder = binder.fold(FRAME_SCOPE_BINDING); - } else { - binder = binder.fold(SCOPE_BINDING); - } - } else { - if (needsFrame) { - binder = binder.fold(FRAME_BINDING); - } else { - binder = binder.fold(SELF_BINDING); - } - }*/ - } - - MethodHandle blockMaker = binder.drop(1, 3) - .append(body) - .invoke(CONSTRUCT_BLOCK); - - return new ConstantCallSite(blockMaker); - } - - static String logMethod(DynamicMethod method) { - return "[#" + method.getSerialNumber() + " " + method.getImplementationClass().getMethodLocation() + "]"; - } - - static String logBlock(Block block) { - return "[" + block.getBody().getFile() + ":" + block.getBody().getLine() + "]"; - } - - private static final Binder BINDING_MAKER_BINDER = Binder.from(Binding.class, ThreadContext.class, IRubyObject.class, DynamicScope.class); - - private static final MethodHandle FRAME_SCOPE_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, IRRuntimeHelpers.class, "newFrameScopeBinding"); - - private static final MethodHandle FRAME_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, Bootstrap.class, "frameBinding"); - public static Binding frameBinding(ThreadContext context, IRubyObject self, DynamicScope scope) { - Frame frame = context.getCurrentFrame().capture(); - return new Binding(self, frame, frame.getVisibility()); - } - - private static final MethodHandle SCOPE_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, Bootstrap.class, "scopeBinding"); - public static Binding scopeBinding(ThreadContext context, IRubyObject self, DynamicScope scope) { - return new Binding(self, scope); - } - - private static final MethodHandle SELF_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, Bootstrap.class, "selfBinding"); - public static Binding selfBinding(ThreadContext context, IRubyObject self, DynamicScope scope) { - return new Binding(self); - } - - private static final MethodHandle CONSTRUCT_BLOCK = Binder.from(Block.class, Binding.class, CompiledIRBlockBody.class).invokeStaticQuiet(LOOKUP, Bootstrap.class, "constructBlock"); - public static Block constructBlock(Binding binding, CompiledIRBlockBody body) throws Throwable { - return new Block(body, binding); - } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/CallInfoBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/CallInfoBootstrap.java new file mode 100644 index 00000000000..d27bda3fda3 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/CallInfoBootstrap.java @@ -0,0 +1,37 @@ +package org.jruby.ir.targets.indy; + +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.runtime.ThreadContext; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static java.lang.invoke.MethodHandles.insertArguments; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.util.CodegenUtils.p; + +public class CallInfoBootstrap { + public static final Handle CALL_INFO_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(CallInfoBootstrap.class), + "callInfoBootstrap", + Bootstrap.BOOTSTRAP_INT_SIG, + false); + + public static CallSite callInfoBootstrap(MethodHandles.Lookup lookup, String name, MethodType type, int callInfo) throws Throwable { + MethodHandle handle; + if (callInfo == 0) { + handle = lookup.findVirtual(ThreadContext.class, "clearCallInfo", methodType(void.class)); + } else { + handle = lookup.findStatic(IRRuntimeHelpers.class, "setCallInfo", methodType(void.class, ThreadContext.class, int.class)); + handle = insertArguments(handle, 1, callInfo); + } + + return new ConstantCallSite(handle); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/CallSiteCacheBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/CallSiteCacheBootstrap.java new file mode 100644 index 00000000000..dd6a5764a22 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/CallSiteCacheBootstrap.java @@ -0,0 +1,44 @@ +package org.jruby.ir.targets.indy; + +import org.jruby.runtime.CallType; +import org.jruby.runtime.callsite.CachingCallSite; +import org.jruby.runtime.callsite.FunctionalCachingCallSite; +import org.jruby.runtime.callsite.MonomorphicCallSite; +import org.jruby.runtime.callsite.VariableCachingCallSite; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static java.lang.invoke.MethodHandles.constant; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class CallSiteCacheBootstrap { + public static final Handle CALLSITE = new Handle( + Opcodes.H_INVOKESTATIC, + p(CallSiteCacheBootstrap.class), + "callSite", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, int.class), + false); + + public static CallSite callSite(MethodHandles.Lookup lookup, String name, MethodType type, String id, int callType) { + return new ConstantCallSite(constant(CachingCallSite.class, callSite(id, callType))); + } + + private static CachingCallSite callSite(String id, int callType) { + switch (CallType.fromOrdinal(callType)) { + case NORMAL: + return new MonomorphicCallSite(id); + case FUNCTIONAL: + return new FunctionalCachingCallSite(id); + case VARIABLE: + return new VariableCachingCallSite(id); + default: + throw new RuntimeException("BUG: Unexpected call type " + callType + " in JVM6 invoke logic"); + } + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/CheckpointSite.java b/core/src/main/java/org/jruby/ir/targets/indy/CheckpointSite.java new file mode 100644 index 00000000000..caba15b302b --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/CheckpointSite.java @@ -0,0 +1,60 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.Ruby; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.opto.Invalidator; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; +import java.lang.invoke.SwitchPoint; + +import static java.lang.invoke.MethodHandles.lookup; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.util.CodegenUtils.p; + +public class CheckpointSite extends MutableCallSite { + public static final Handle CHECKPOINT_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(CheckpointSite.class), + "checkpointBootstrap", + Bootstrap.BOOTSTRAP_BARE_SIG, + false); + + public CheckpointSite(MethodType type) { + super(type); + } + + public static CallSite checkpointBootstrap(MethodHandles.Lookup lookup, String name, MethodType type) throws Throwable { + CheckpointSite site = new CheckpointSite(type); + MethodHandle handle = lookup.findVirtual(CheckpointSite.class, "checkpointFallback", methodType(void.class, ThreadContext.class)); + + handle = handle.bindTo(site); + site.setTarget(handle); + + return site; + } + + public void checkpointFallback(ThreadContext context) throws Throwable { + Ruby runtime = context.runtime; + Invalidator invalidator = runtime.getCheckpointInvalidator(); + + MethodHandle target = Binder + .from(void.class, ThreadContext.class) + .nop(); + MethodHandle fallback = lookup().findVirtual(CheckpointSite.class, "checkpointFallback", methodType(void.class, ThreadContext.class)); + fallback = fallback.bindTo(this); + + target = ((SwitchPoint)invalidator.getData()).guardWithTest(target, fallback); + + this.setTarget(target); + + // poll for events once since we've ended up back in fallback + context.pollThreadEvents(); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/ConstructBlockBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/ConstructBlockBootstrap.java new file mode 100644 index 00000000000..aa54a1046b3 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/ConstructBlockBootstrap.java @@ -0,0 +1,104 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.parser.StaticScope; +import org.jruby.runtime.Binding; +import org.jruby.runtime.Block; +import org.jruby.runtime.CompiledIRBlockBody; +import org.jruby.runtime.DynamicScope; +import org.jruby.runtime.Frame; +import org.jruby.runtime.Helpers; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class ConstructBlockBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + private static final Binder BINDING_MAKER_BINDER = Binder.from(Binding.class, ThreadContext.class, IRubyObject.class, DynamicScope.class); + private static final MethodHandle SELF_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, ConstructBlockBootstrap.class, "selfBinding"); + private static final MethodHandle SCOPE_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, ConstructBlockBootstrap.class, "scopeBinding"); + private static final MethodHandle FRAME_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, ConstructBlockBootstrap.class, "frameBinding"); + private static final MethodHandle FRAME_SCOPE_BINDING = BINDING_MAKER_BINDER.invokeStaticQuiet(LOOKUP, IRRuntimeHelpers.class, "newFrameScopeBinding"); + private static final MethodHandle CONSTRUCT_BLOCK = Binder.from(Block.class, Binding.class, CompiledIRBlockBody.class).invokeStaticQuiet(LOOKUP, ConstructBlockBootstrap.class, "constructBlock"); + public static final Handle PREPARE_BLOCK_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(ConstructBlockBootstrap.class), + "prepareBlock", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, + MethodType.class, MethodHandle.class, MethodHandle.class, MethodHandle.class, MethodHandle.class, String.class, long.class, + String.class, int.class, String.class), + false); + + public static CallSite prepareBlock(MethodHandles.Lookup lookup, String name, MethodType type, + MethodHandle bodyHandle, MethodHandle scopeHandle, MethodHandle setScopeHandle, MethodHandle parentHandle, String scopeDescriptor, long encodedSignature, + String file, int line, String encodedArgumentDescriptors + ) throws Throwable { + StaticScope staticScope = (StaticScope) scopeHandle.invokeExact(); + + if (staticScope == null) { + staticScope = Helpers.restoreScope(scopeDescriptor, (StaticScope) parentHandle.invokeExact()); + setScopeHandle.invokeExact(staticScope); + } + + CompiledIRBlockBody body = new CompiledIRBlockBody(bodyHandle, staticScope, file, line, encodedArgumentDescriptors, encodedSignature); + + Binder binder = Binder.from(type); + + binder = binder.fold(FRAME_SCOPE_BINDING); + + // This optimization can't happen until we can see into the method we're calling to know if it reifies the block + if (false) { + /* + if (needsBinding) { + if (needsFrame) { + FullInterpreterContext fic = scope.getExecutionContext(); + if (fic.needsBinding()) { + if (fic.needsFrame()) { + binder = binder.fold(FRAME_SCOPE_BINDING); + } else { + binder = binder.fold(SCOPE_BINDING); + } + } else { + if (needsFrame) { + binder = binder.fold(FRAME_BINDING); + } else { + binder = binder.fold(SELF_BINDING); + } + }*/ + } + + MethodHandle blockMaker = binder.drop(1, 3) + .append(body) + .invoke(CONSTRUCT_BLOCK); + + return new ConstantCallSite(blockMaker); + } + + public static Binding frameBinding(ThreadContext context, IRubyObject self, DynamicScope scope) { + Frame frame = context.getCurrentFrame().capture(); + return new Binding(self, frame, frame.getVisibility()); + } + + public static Binding scopeBinding(ThreadContext context, IRubyObject self, DynamicScope scope) { + return new Binding(self, scope); + } + + public static Binding selfBinding(ThreadContext context, IRubyObject self, DynamicScope scope) { + return new Binding(self); + } + + public static Block constructBlock(Binding binding, CompiledIRBlockBody body) throws Throwable { + return new Block(body, binding); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/CoverageSite.java b/core/src/main/java/org/jruby/ir/targets/indy/CoverageSite.java new file mode 100644 index 00000000000..3f1b525fc2b --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/CoverageSite.java @@ -0,0 +1,44 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.runtime.ThreadContext; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; + +import static java.lang.invoke.MethodHandles.insertArguments; +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class CoverageSite { + public static final Handle COVER_LINE_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(CoverageSite.class), + "coverLineBootstrap", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, String.class, int.class, int.class), + false); + + public static CallSite coverLineBootstrap(MethodHandles.Lookup lookup, String name, MethodType type, String filename, int line, int oneshot) throws Throwable { + MutableCallSite site = new MutableCallSite(type); + MethodHandle handle = lookup.findStatic(CoverageSite.class, "coverLineFallback", methodType(void.class, MutableCallSite.class, ThreadContext.class, String.class, int.class, boolean.class)); + + handle = handle.bindTo(site); + handle = insertArguments(handle, 1, filename, line, oneshot != 0); + site.setTarget(handle); + + return site; + } + + public static void coverLineFallback(MutableCallSite site, ThreadContext context, String filename, int line, boolean oneshot) throws Throwable { + IRRuntimeHelpers.updateCoverage(context, filename, line); + + if (oneshot) site.setTarget(Binder.from(void.class, ThreadContext.class).dropAll().nop()); + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/HeapVariableBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/HeapVariableBootstrap.java new file mode 100644 index 00000000000..38b218e4042 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/HeapVariableBootstrap.java @@ -0,0 +1,84 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.scope.DynamicScopeGenerator; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static java.lang.invoke.MethodType.methodType; +import static org.jruby.runtime.Helpers.arrayOf; +import static org.jruby.util.CodegenUtils.p; + +public class HeapVariableBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + public static final Handle GET_HEAP_LOCAL_OR_NIL_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(HeapVariableBootstrap.class), + "getHeapLocalOrNilBootstrap", + Bootstrap.BOOTSTRAP_INT_INT_SIG, + false); + public static final Handle GET_HEAP_LOCAL_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(HeapVariableBootstrap.class), + "getHeapLocalBootstrap", + Bootstrap.BOOTSTRAP_INT_INT_SIG, + false); + + public static CallSite getHeapLocalBootstrap(MethodHandles.Lookup lookup, String name, MethodType type, int depth, int location) throws Throwable { + // no null checking needed for method bodies + MethodHandle getter; + Binder binder = Binder + .from(type); + + if (depth == 0) { + if (location < DynamicScopeGenerator.SPECIALIZED_GETS.size()) { + getter = binder.invokeVirtualQuiet(LOOKUP, DynamicScopeGenerator.SPECIALIZED_GETS.get(location)); + } else { + getter = binder + .insert(1, location) + .invokeVirtualQuiet(LOOKUP, "getValueDepthZero"); + } + } else { + getter = binder + .insert(1, arrayOf(int.class, int.class), location, depth) + .invokeVirtualQuiet(LOOKUP, "getValue"); + } + + ConstantCallSite site = new ConstantCallSite(getter); + + return site; + } + + public static CallSite getHeapLocalOrNilBootstrap(MethodHandles.Lookup lookup, String name, MethodType type, int depth, int location) throws Throwable { + MethodHandle getter; + Binder binder = Binder + .from(type) + .filter(1, LiteralValueBootstrap.contextValue(lookup, "nil", methodType(IRubyObject.class, ThreadContext.class)).dynamicInvoker()); + + if (depth == 0) { + if (location < DynamicScopeGenerator.SPECIALIZED_GETS_OR_NIL.size()) { + getter = binder.invokeVirtualQuiet(LOOKUP, DynamicScopeGenerator.SPECIALIZED_GETS_OR_NIL.get(location)); + } else { + getter = binder + .insert(1, location) + .invokeVirtualQuiet(LOOKUP, "getValueDepthZeroOrNil"); + } + } else { + getter = binder + .insert(1, arrayOf(int.class, int.class), location, depth) + .invokeVirtualQuiet(LOOKUP, "getValueOrNil"); + } + + ConstantCallSite site = new ConstantCallSite(getter); + + return site; + } +} diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyBlockCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyBlockCompiler.java index a5d8662f8a0..f79a88e084f 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyBlockCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyBlockCompiler.java @@ -49,6 +49,6 @@ public void prepareBlock(IRClosure closure, String parentScopeField, Handle hand long encodedSignature = signature.encode(); compiler.adapter.invokedynamic(handle.getName(), sig(Block.class, ThreadContext.class, IRubyObject.class, DynamicScope.class), - Bootstrap.prepareBlock(), handle, scopeHandle, setScopeHandle, parentScopeHandle, scopeDescriptor, encodedSignature, file, line, encodedArgumentDescriptors); + ConstructBlockBootstrap.PREPARE_BLOCK_BOOTSTRAP, handle, scopeHandle, setScopeHandle, parentScopeHandle, scopeDescriptor, encodedSignature, file, line, encodedArgumentDescriptors); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyCheckpointCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyCheckpointCompiler.java index 86c42a211b9..2b375de36ce 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyCheckpointCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyCheckpointCompiler.java @@ -18,6 +18,6 @@ public void checkpoint() { compiler.adapter.invokedynamic( "checkpoint", sig(void.class, ThreadContext.class), - Bootstrap.checkpointHandle()); + CheckpointSite.CHECKPOINT_BOOTSTRAP); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java index 3b22b36698c..3c980ff2719 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyInvocationCompiler.java @@ -16,6 +16,7 @@ import org.jruby.runtime.MethodIndex; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.invokedynamic.MathLinker; import org.jruby.util.CodegenUtils; import org.jruby.util.JavaNameMangler; @@ -78,7 +79,7 @@ public void invokeOtherOneFixnum(String file, CallBase call, long fixnum) { compiler.adapter.invokedynamic( "fixnumOperator:" + JavaNameMangler.mangleMethodName(id), signature, - Bootstrap.getFixnumOperatorHandle(), + MathLinker.FIXNUM_OPERATOR_BOOTSTRAP, fixnum, call.getCallType().ordinal(), file, @@ -99,7 +100,7 @@ public void invokeOtherOneFloat(String file, CallBase call, double flote) { compiler.adapter.invokedynamic( "floatOperator:" + JavaNameMangler.mangleMethodName(id), signature, - Bootstrap.getFloatOperatorHandle(), + MathLinker.FLOAT_OPERATOR_BOOTSTRAP, flote, call.getCallType().ordinal(), file, @@ -212,6 +213,6 @@ public void asString(AsStringInstr call, String scopeFieldName, String file) { @Override public void setCallInfo(int flags) { compiler.loadContext(); - compiler.adapter.invokedynamic("callInfo", sig(void.class, ThreadContext.class), Bootstrap.callInfoHandle(), flags); + compiler.adapter.invokedynamic("callInfo", sig(void.class, ThreadContext.class), CallInfoBootstrap.CALL_INFO_BOOTSTRAP, flags); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyLocalVariableCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyLocalVariableCompiler.java index eb5fa363204..47aa232d5b5 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyLocalVariableCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyLocalVariableCompiler.java @@ -26,12 +26,12 @@ public void getHeapLocal(int depth, int location) { normalLocalVariableCompiler.getHeapLocal(depth, location); return; } - compiler.adapter.invokedynamic("getHeapLocal", sig(IRubyObject.class, DynamicScope.class), Bootstrap.getHeapLocalHandle(), depth, location); + compiler.adapter.invokedynamic("getHeapLocal", sig(IRubyObject.class, DynamicScope.class), HeapVariableBootstrap.GET_HEAP_LOCAL_BOOTSTRAP, depth, location); } @Override public void getHeapLocalOrNil(int depth, int location) { compiler.loadContext(); - compiler.adapter.invokedynamic("getHeapLocalOrNil", sig(IRubyObject.class, DynamicScope.class, ThreadContext.class), Bootstrap.getHeapLocalOrNilHandle(), depth, location); + compiler.adapter.invokedynamic("getHeapLocalOrNil", sig(IRubyObject.class, DynamicScope.class, ThreadContext.class), HeapVariableBootstrap.GET_HEAP_LOCAL_OR_NIL_BOOTSTRAP, depth, location); } } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java b/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java index 014076e9641..08e414b9986 100644 --- a/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java +++ b/core/src/main/java/org/jruby/ir/targets/indy/IndyValueCompiler.java @@ -155,7 +155,7 @@ public void pushCallSite(String className, String siteName, String scopeFieldNam if (!specialSite) { // use indy to cache the site object - method.invokedynamic("callSite", sig(CachingCallSite.class), Bootstrap.CALLSITE, call.getId(), callType.ordinal()); + method.invokedynamic("callSite", sig(CachingCallSite.class), CallSiteCacheBootstrap.CALLSITE, call.getId(), callType.ordinal()); return; } diff --git a/core/src/main/java/org/jruby/ir/targets/indy/MetaClassBootstrap.java b/core/src/main/java/org/jruby/ir/targets/indy/MetaClassBootstrap.java new file mode 100644 index 00000000000..45e4f6dd417 --- /dev/null +++ b/core/src/main/java/org/jruby/ir/targets/indy/MetaClassBootstrap.java @@ -0,0 +1,57 @@ +package org.jruby.ir.targets.indy; + +import com.headius.invokebinder.Binder; +import org.jruby.internal.runtime.methods.DynamicMethod; +import org.jruby.ir.JIT; +import org.jruby.ir.runtime.IRRuntimeHelpers; +import org.jruby.parser.StaticScope; +import org.jruby.runtime.Helpers; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static java.lang.invoke.MethodHandles.insertArguments; +import static org.jruby.util.CodegenUtils.p; +import static org.jruby.util.CodegenUtils.sig; + +public class MetaClassBootstrap { + private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); + + public static final Handle OPEN_META_CLASS = new Handle( + Opcodes.H_INVOKESTATIC, + p(MetaClassBootstrap.class), + "openMetaClass", + sig(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, MethodHandle.class, MethodHandle.class, MethodHandle.class, int.class, int.class, int.class), + false); + private static final MethodHandle OPEN_META_CLASS_HANDLE = + Binder + .from(DynamicMethod.class, ThreadContext.class, IRubyObject.class, String.class, StaticScope.class, MethodHandle.class, StaticScope.class, MethodHandle.class, int.class, boolean.class, boolean.class) + .invokeStaticQuiet(LOOKUP, MetaClassBootstrap.class, "openMetaClass"); + + @JIT + public static CallSite openMetaClass(MethodHandles.Lookup lookup, String name, MethodType type, MethodHandle body, MethodHandle scope, MethodHandle setScope, int line, int dynscopeEliminated, int refinements) { + try { + StaticScope staticScope = (StaticScope) scope.invokeExact(); + return new ConstantCallSite(insertArguments(OPEN_META_CLASS_HANDLE, 4, body, staticScope, setScope, line, dynscopeEliminated == 1 ? true : false, refinements == 1 ? true : false)); + } catch (Throwable t) { + Helpers.throwException(t); + return null; + } + } + + @JIT + public static DynamicMethod openMetaClass(ThreadContext context, IRubyObject object, String descriptor, StaticScope parent, MethodHandle body, StaticScope scope, MethodHandle setScope, int line, boolean dynscopeEliminated, boolean refinements) throws Throwable { + if (scope == null) { + scope = Helpers.restoreScope(descriptor, parent); + setScope.invokeExact(scope); + } + return IRRuntimeHelpers.newCompiledMetaClass(context, body, scope, object, line, dynscopeEliminated, refinements); + } +} diff --git a/core/src/main/java/org/jruby/runtime/invokedynamic/MathLinker.java b/core/src/main/java/org/jruby/runtime/invokedynamic/MathLinker.java index 5c02c89e8ac..02d1cd816be 100644 --- a/core/src/main/java/org/jruby/runtime/invokedynamic/MathLinker.java +++ b/core/src/main/java/org/jruby/runtime/invokedynamic/MathLinker.java @@ -29,13 +29,12 @@ import java.lang.invoke.CallSite; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.lang.invoke.SwitchPoint; -import java.util.List; import static java.lang.invoke.MethodHandles.*; import static java.lang.invoke.MethodType.*; +import static org.jruby.util.CodegenUtils.p; import com.headius.invokebinder.Binder; import org.jruby.Ruby; @@ -43,6 +42,7 @@ import org.jruby.RubyFixnum; import org.jruby.RubyFloat; import org.jruby.RubyNumeric; +import org.jruby.ir.targets.indy.Bootstrap; import org.jruby.ir.targets.simple.NormalInvokeSite; import org.jruby.runtime.CallType; import org.jruby.runtime.MethodIndex; @@ -54,11 +54,25 @@ import org.jruby.util.cli.Options; import org.jruby.util.log.Logger; import org.jruby.util.log.LoggerFactory; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; public class MathLinker { private static final Logger LOG = LoggerFactory.getLogger(MathLinker.class); public static final Lookup LOOKUP = lookup(); + public static final Handle FIXNUM_OPERATOR_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(MathLinker.class), + "fixnumOperatorBootstrap", + Bootstrap.BOOTSTRAP_LONG_STRING_INT_SIG, + false); + public static final Handle FLOAT_OPERATOR_BOOTSTRAP = new Handle( + Opcodes.H_INVOKESTATIC, + p(MathLinker.class), + "floatOperatorBootstrap", + Bootstrap.BOOTSTRAP_DOUBLE_STRING_INT_SIG, + false); static { // enable DEBUG output if (Options.INVOKEDYNAMIC_LOG_BINDING.load()) LOG.setDebugEnable(true); @@ -372,5 +386,4 @@ private static CacheEntry searchWithCache(final String operator, IRubyObject cal } return entry; } - } From 83af68ee2e852150e9b5426d34b2e8514663363e Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Sun, 29 Oct 2023 06:41:16 +0100 Subject: [PATCH 44/56] [fix] infinite loop calling Java super from Ruby (#7990) --- .../jruby/java/proxies/ConcreteJavaProxy.java | 1 + .../java/org/jruby/javasupport/TestJava.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/core/src/main/java/org/jruby/java/proxies/ConcreteJavaProxy.java b/core/src/main/java/org/jruby/java/proxies/ConcreteJavaProxy.java index d3dcc16716d..759b5e1d06b 100644 --- a/core/src/main/java/org/jruby/java/proxies/ConcreteJavaProxy.java +++ b/core/src/main/java/org/jruby/java/proxies/ConcreteJavaProxy.java @@ -397,6 +397,7 @@ private static boolean isClassOrIncludedPrependedModule(final RubyModule methodS RubyClass candidate = klass.getSuperClass(); while (candidate != null && (candidate.isIncluded() || candidate.isPrepended())) { // up till 'real' superclass if (candidate == klass) return true; + candidate = candidate.getSuperClass(); } return false; diff --git a/core/src/test/java/org/jruby/javasupport/TestJava.java b/core/src/test/java/org/jruby/javasupport/TestJava.java index ee49e27acd4..dbf2864c1aa 100644 --- a/core/src/test/java/org/jruby/javasupport/TestJava.java +++ b/core/src/test/java/org/jruby/javasupport/TestJava.java @@ -1,6 +1,7 @@ package org.jruby.javasupport; import java.lang.reflect.Method; +import java.text.SimpleDateFormat; import org.jruby.*; import org.jruby.exceptions.RaiseException; @@ -117,4 +118,23 @@ public void testJavaConstructorExceptionHandling() throws Exception { } } + @Test + public void testOverrideNewOnConcreteJavaProxySubClassRegression() { + String script = + "class FormatImpl < java.text.SimpleDateFormat\n" + + " include Enumerable\n" + + " public_class_method :new\n" + + " class << self\n" + + " def new(thread_provider: true)\n" + + " super()\n" + + " end\n" + + " end\n" + + "end\n"; + + final Ruby runtime = Ruby.newInstance(); + runtime.evalScriptlet(script); + + assertNotNull(runtime.evalScriptlet("FormatImpl.new")); // used to cause an infinite loop + assertTrue(runtime.evalScriptlet("FormatImpl.new").toJava(Object.class) instanceof SimpleDateFormat); + } } From b41a664dd18f483c827dd5d0965ad892172e7158 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 30 Oct 2023 13:51:20 +0100 Subject: [PATCH 45/56] Update to ruby/spec@bd7017f --- spec/ruby/.rubocop.yml | 4 + spec/ruby/CONTRIBUTING.md | 1 + spec/ruby/README.md | 6 + spec/ruby/command_line/dash_a_spec.rb | 4 +- spec/ruby/command_line/dash_l_spec.rb | 8 +- spec/ruby/command_line/dash_n_spec.rb | 8 +- spec/ruby/command_line/dash_p_spec.rb | 4 +- spec/ruby/command_line/dash_upper_f_spec.rb | 2 +- spec/ruby/command_line/rubyopt_spec.rb | 14 +- spec/ruby/core/argf/readpartial_spec.rb | 2 +- spec/ruby/core/array/bsearch_index_spec.rb | 4 - spec/ruby/core/dir/glob_spec.rb | 10 +- spec/ruby/core/dir/home_spec.rb | 27 ++- .../core/exception/detailed_message_spec.rb | 12 +- spec/ruby/core/exception/full_message_spec.rb | 40 +++- spec/ruby/core/file/flock_spec.rb | 4 +- spec/ruby/core/file/new_spec.rb | 6 + spec/ruby/core/file/shared/fnmatch.rb | 40 +++- spec/ruby/core/io/copy_stream_spec.rb | 33 ++- spec/ruby/core/io/gets_spec.rb | 6 + spec/ruby/core/io/new_spec.rb | 6 + spec/ruby/core/io/open_spec.rb | 13 ++ spec/ruby/core/io/pread_spec.rb | 77 +++++++ spec/ruby/core/io/pwrite_spec.rb | 30 ++- spec/ruby/core/io/read_spec.rb | 18 ++ spec/ruby/core/io/shared/each.rb | 4 - spec/ruby/core/kernel/exec_spec.rb | 4 +- spec/ruby/core/kernel/lambda_spec.rb | 72 +++--- spec/ruby/core/kernel/printf_spec.rb | 7 + spec/ruby/core/kernel/shared/load.rb | 38 ++-- spec/ruby/core/kernel/shared/require.rb | 28 +++ spec/ruby/core/kernel/sleep_spec.rb | 1 - spec/ruby/core/kernel/sprintf_spec.rb | 16 ++ spec/ruby/core/matchdata/values_at_spec.rb | 4 +- spec/ruby/core/method/super_method_spec.rb | 10 +- spec/ruby/core/module/define_method_spec.rb | 8 +- spec/ruby/core/proc/lambda_spec.rb | 8 +- spec/ruby/core/process/constants_spec.rb | 8 +- spec/ruby/core/process/exec_spec.rb | 42 ++-- spec/ruby/core/process/times_spec.rb | 27 --- spec/ruby/core/regexp/union_spec.rb | 51 +++-- spec/ruby/core/signal/signame_spec.rb | 12 + spec/ruby/core/signal/trap_spec.rb | 19 ++ spec/ruby/core/string/start_with_spec.rb | 9 +- .../ruby/core/thread/native_thread_id_spec.rb | 9 +- .../core/unboundmethod/super_method_spec.rb | 10 +- .../core/warning/element_reference_spec.rb | 8 +- spec/ruby/fixtures/code/d/load_fixture.rb.rb | 1 + spec/ruby/language/alias_spec.rb | 2 +- spec/ruby/language/case_spec.rb | 95 ++++---- spec/ruby/language/delegation_spec.rb | 38 ++-- spec/ruby/library/bigdecimal/to_s_spec.rb | 24 +- .../library/cgi/escapeURIComponent_spec.rb | 57 +++++ spec/ruby/library/cgi/initialize_spec.rb | 2 +- spec/ruby/library/datetime/rfc2822_spec.rb | 4 + .../library/openssl/digest/append_spec.rb | 6 + .../openssl/digest/block_length_spec.rb | 44 ++++ .../openssl/digest/digest_length_spec.rb | 44 ++++ .../openssl/{ => digest}/digest_spec.rb | 13 +- .../library/openssl/digest/initialize_spec.rb | 141 ++++++++++++ spec/ruby/library/openssl/digest/name_spec.rb | 16 ++ .../ruby/library/openssl/digest/reset_spec.rb | 36 +++ .../library/openssl/digest/shared/update.rb | 123 +++++++++++ .../library/openssl/digest/update_spec.rb | 6 + .../library/openssl/kdf/pbkdf2_hmac_spec.rb | 184 ++++++++++++++++ spec/ruby/library/openssl/kdf/scrypt_spec.rb | 207 ++++++++++++++++++ .../socket/ipsocket/getaddress_spec.rb | 2 +- .../ruby/library/socket/tcpserver/new_spec.rb | 6 + .../socket/tcpsocket/initialize_spec.rb | 21 ++ .../library/socket/tcpsocket/shared/new.rb | 6 + .../ruby/library/socket/udpsocket/new_spec.rb | 6 + .../socket/unixserver/accept_nonblock_spec.rb | 7 +- .../library/socket/unixserver/accept_spec.rb | 2 +- .../library/socket/unixserver/for_fd_spec.rb | 2 +- .../library/socket/unixserver/new_spec.rb | 12 +- .../library/socket/unixserver/open_spec.rb | 6 +- .../library/socket/unixserver/shared/new.rb | 26 +-- .../library/socket/unixsocket/addr_spec.rb | 5 +- .../library/socket/unixsocket/inspect_spec.rb | 4 +- .../socket/unixsocket/local_address_spec.rb | 2 - .../library/socket/unixsocket/new_spec.rb | 12 +- .../library/socket/unixsocket/open_spec.rb | 10 +- .../library/socket/unixsocket/pair_spec.rb | 5 +- .../unixsocket/partially_closable_spec.rb | 4 +- .../library/socket/unixsocket/path_spec.rb | 6 +- .../socket/unixsocket/peeraddr_spec.rb | 6 +- .../library/socket/unixsocket/recv_io_spec.rb | 7 +- .../socket/unixsocket/recvfrom_spec.rb | 7 +- .../library/socket/unixsocket/send_io_spec.rb | 7 +- .../library/socket/unixsocket/shared/new.rb | 28 ++- spec/ruby/library/stringio/new_spec.rb | 6 +- spec/ruby/optional/capi/encoding_spec.rb | 30 +++ spec/ruby/optional/capi/ext/encoding_spec.c | 15 ++ spec/ruby/optional/capi/util_spec.rb | 13 ++ spec/ruby/shared/string/start_with.rb | 8 +- 95 files changed, 1709 insertions(+), 379 deletions(-) create mode 100644 spec/ruby/fixtures/code/d/load_fixture.rb.rb create mode 100644 spec/ruby/library/cgi/escapeURIComponent_spec.rb create mode 100644 spec/ruby/library/openssl/digest/append_spec.rb create mode 100644 spec/ruby/library/openssl/digest/block_length_spec.rb create mode 100644 spec/ruby/library/openssl/digest/digest_length_spec.rb rename spec/ruby/library/openssl/{ => digest}/digest_spec.rb (84%) create mode 100644 spec/ruby/library/openssl/digest/initialize_spec.rb create mode 100644 spec/ruby/library/openssl/digest/name_spec.rb create mode 100644 spec/ruby/library/openssl/digest/reset_spec.rb create mode 100644 spec/ruby/library/openssl/digest/shared/update.rb create mode 100644 spec/ruby/library/openssl/digest/update_spec.rb create mode 100644 spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb create mode 100644 spec/ruby/library/openssl/kdf/scrypt_spec.rb diff --git a/spec/ruby/.rubocop.yml b/spec/ruby/.rubocop.yml index 99209b607bd..9ad57dd51cb 100644 --- a/spec/ruby/.rubocop.yml +++ b/spec/ruby/.rubocop.yml @@ -33,6 +33,10 @@ Lint/AssignmentInCondition: Lint/BooleanSymbol: Enabled: false +Lint/DeprecatedOpenSSLConstant: + Exclude: + - library/openssl/digest/**/*.rb + Lint/InterpolationCheck: Enabled: false diff --git a/spec/ruby/CONTRIBUTING.md b/spec/ruby/CONTRIBUTING.md index 4a1e1d46bdb..c82eb5ea4fa 100644 --- a/spec/ruby/CONTRIBUTING.md +++ b/spec/ruby/CONTRIBUTING.md @@ -179,6 +179,7 @@ In case there is a bug in MRI and the fix will be backported to previous version If it is not backported or not likely, use `ruby_version_is` instead. First, file a bug at https://bugs.ruby-lang.org/. The problem is `ruby_bug` would make non-MRI implementations fail this spec while MRI itself does not pass it, so it should only be used if the bug is/will be fixed and backported. +Otherwise, non-MRI implementations would have to choose between being incompatible with the latest release of MRI to pass the spec or fail the spec, both which make no sense. ```ruby ruby_bug '#13669', ''...'3.2' do diff --git a/spec/ruby/README.md b/spec/ruby/README.md index 536388e7c89..115392835f9 100644 --- a/spec/ruby/README.md +++ b/spec/ruby/README.md @@ -128,6 +128,12 @@ MSpec can automatically add new top-level constants in this file with: $ CHECK_LEAKS=save mspec ../mspec/bin/mspec file +### Running Specs on S390x CPU Architecture + +Run the specs with `DFLTCC=0` if you see failing specs related to the zlib library on s390x CPU architecture. The failures can happen with the zlib library applying the patch madler/zlib#410 to enable the deflate algorithm producing a different compressed byte stream. + + $ DFLTCC=0 ../mspec/bin/mspec + ### Contributing and Writing Specs See [CONTRIBUTING.md](https://github.com/ruby/spec/blob/master/CONTRIBUTING.md) for documentation about contributing and writing specs (guards, matchers, etc). diff --git a/spec/ruby/command_line/dash_a_spec.rb b/spec/ruby/command_line/dash_a_spec.rb index 9ea135dc76f..43d923ce164 100644 --- a/spec/ruby/command_line/dash_a_spec.rb +++ b/spec/ruby/command_line/dash_a_spec.rb @@ -6,13 +6,13 @@ end it "runs the code in loop conditional on Kernel.gets()" do - ruby_exe("puts $F.last", options: "-n -a", escape: true, + ruby_exe("puts $F.last", options: "-n -a", args: " < #{@names}").should == "jones\nfield\ngrey\n" end it "sets $-a" do - ruby_exe("puts $-a", options: "-n -a", escape: true, + ruby_exe("puts $-a", options: "-n -a", args: " < #{@names}").should == "true\ntrue\ntrue\n" end diff --git a/spec/ruby/command_line/dash_l_spec.rb b/spec/ruby/command_line/dash_l_spec.rb index 5c1d3cf4cd9..44a98445f37 100644 --- a/spec/ruby/command_line/dash_l_spec.rb +++ b/spec/ruby/command_line/dash_l_spec.rb @@ -6,25 +6,25 @@ end it "chomps lines with default separator" do - ruby_exe('puts $_.end_with?("\n")', options: "-n -l", escape: true, + ruby_exe('puts $_.end_with?("\n")', options: "-n -l", args: " < #{@names}").should == "false\nfalse\nfalse\n" end it "chomps last line based on $/" do - ruby_exe('BEGIN { $/ = "ones\n" }; puts $_', options: "-W0 -n -l", escape: true, + ruby_exe('BEGIN { $/ = "ones\n" }; puts $_', options: "-W0 -n -l", args: " < #{@names}").should == "alice j\nbob field\njames grey\n" end it "sets $\\ to the value of $/" do - ruby_exe("puts $\\ == $/", options: "-W0 -n -l", escape: true, + ruby_exe("puts $\\ == $/", options: "-W0 -n -l", args: " < #{@names}").should == "true\ntrue\ntrue\n" end it "sets $-l" do - ruby_exe("puts $-l", options: "-n -l", escape: true, + ruby_exe("puts $-l", options: "-n -l", args: " < #{@names}").should == "true\ntrue\ntrue\n" end diff --git a/spec/ruby/command_line/dash_n_spec.rb b/spec/ruby/command_line/dash_n_spec.rb index 9d331d6065d..1dd9379259b 100644 --- a/spec/ruby/command_line/dash_n_spec.rb +++ b/spec/ruby/command_line/dash_n_spec.rb @@ -6,19 +6,19 @@ end it "runs the code in loop conditional on Kernel.gets()" do - ruby_exe("puts $_", options: "-n", escape: true, + ruby_exe("puts $_", options: "-n", args: " < #{@names}").should == "alice\nbob\njames\n" end it "only evaluates BEGIN blocks once" do - ruby_exe("BEGIN { puts \"hi\" }; puts $_", options: "-n", escape: true, + ruby_exe("BEGIN { puts \"hi\" }; puts $_", options: "-n", args: " < #{@names}").should == "hi\nalice\nbob\njames\n" end it "only evaluates END blocks once" do - ruby_exe("puts $_; END {puts \"bye\"}", options: "-n", escape: true, + ruby_exe("puts $_; END {puts \"bye\"}", options: "-n", args: " < #{@names}").should == "alice\nbob\njames\nbye\n" end @@ -29,7 +29,7 @@ $total += 1 END { puts $total } script - ruby_exe(script, options: "-n", escape: true, + ruby_exe(script, options: "-n", args: " < #{@names}").should == "3\n" end diff --git a/spec/ruby/command_line/dash_p_spec.rb b/spec/ruby/command_line/dash_p_spec.rb index 39827c3868c..967e3796de2 100644 --- a/spec/ruby/command_line/dash_p_spec.rb +++ b/spec/ruby/command_line/dash_p_spec.rb @@ -6,13 +6,13 @@ end it "runs the code in loop conditional on Kernel.gets() and prints $_" do - ruby_exe("$_ = $_.upcase", options: "-p", escape: true, + ruby_exe("$_ = $_.upcase", options: "-p", args: " < #{@names}").should == "ALICE\nBOB\nJAMES\n" end it "sets $-p" do - ruby_exe("$_ = $-p", options: "-p", escape: true, + ruby_exe("$_ = $-p", options: "-p", args: " < #{@names}").should == "truetruetrue" end diff --git a/spec/ruby/command_line/dash_upper_f_spec.rb b/spec/ruby/command_line/dash_upper_f_spec.rb index 967acc2ece1..5c10a7140d1 100644 --- a/spec/ruby/command_line/dash_upper_f_spec.rb +++ b/spec/ruby/command_line/dash_upper_f_spec.rb @@ -6,7 +6,7 @@ end it "specifies the field separator pattern for -a" do - ruby_exe("puts $F[0]", options: "-naF:", escape: true, + ruby_exe("puts $F[0]", options: "-naF:", args: " < #{@passwd}").should == "nobody\nroot\ndaemon\n" end diff --git a/spec/ruby/command_line/rubyopt_spec.rb b/spec/ruby/command_line/rubyopt_spec.rb index bbea4d557d1..734db8d519e 100644 --- a/spec/ruby/command_line/rubyopt_spec.rb +++ b/spec/ruby/command_line/rubyopt_spec.rb @@ -11,14 +11,14 @@ it "adds the -I path to $LOAD_PATH" do ENV["RUBYOPT"] = "-Ioptrubyspecincl" - result = ruby_exe("puts $LOAD_PATH.grep(/byspecin/)", escape: true) + result = ruby_exe("puts $LOAD_PATH.grep(/byspecin/)") result.chomp[-15..-1].should == "optrubyspecincl" end it "sets $DEBUG to true for '-d'" do ENV["RUBYOPT"] = '-d' command = %[puts "value of $DEBUG is \#{$DEBUG}"] - result = ruby_exe(command, escape: true, args: "2>&1") + result = ruby_exe(command, args: "2>&1") result.should =~ /value of \$DEBUG is true/ end @@ -36,27 +36,27 @@ it "sets $VERBOSE to true for '-w'" do ENV["RUBYOPT"] = '-w' - ruby_exe("p $VERBOSE", escape: true).chomp.should == "true" + ruby_exe("p $VERBOSE").chomp.should == "true" end it "sets $VERBOSE to true for '-W'" do ENV["RUBYOPT"] = '-W' - ruby_exe("p $VERBOSE", escape: true).chomp.should == "true" + ruby_exe("p $VERBOSE").chomp.should == "true" end it "sets $VERBOSE to nil for '-W0'" do ENV["RUBYOPT"] = '-W0' - ruby_exe("p $VERBOSE", escape: true).chomp.should == "nil" + ruby_exe("p $VERBOSE").chomp.should == "nil" end it "sets $VERBOSE to false for '-W1'" do ENV["RUBYOPT"] = '-W1' - ruby_exe("p $VERBOSE", escape: true).chomp.should == "false" + ruby_exe("p $VERBOSE").chomp.should == "false" end it "sets $VERBOSE to true for '-W2'" do ENV["RUBYOPT"] = '-W2' - ruby_exe("p $VERBOSE", escape: true).chomp.should == "true" + ruby_exe("p $VERBOSE").chomp.should == "true" end it "suppresses deprecation warnings for '-W:no-deprecated'" do diff --git a/spec/ruby/core/argf/readpartial_spec.rb b/spec/ruby/core/argf/readpartial_spec.rb index 5e284b34233..bbc8831131d 100644 --- a/spec/ruby/core/argf/readpartial_spec.rb +++ b/spec/ruby/core/argf/readpartial_spec.rb @@ -69,7 +69,7 @@ print ARGF.readpartial(#{@stdin.size}) ARGF.readpartial(1) rescue print $!.class STR - stdin = ruby_exe(ruby_str, args: "< #{@stdin_name}", escape: true) + stdin = ruby_exe(ruby_str, args: "< #{@stdin_name}") stdin.should == @stdin + "EOFError" end end diff --git a/spec/ruby/core/array/bsearch_index_spec.rb b/spec/ruby/core/array/bsearch_index_spec.rb index df2c7c098e2..94d85b37f33 100644 --- a/spec/ruby/core/array/bsearch_index_spec.rb +++ b/spec/ruby/core/array/bsearch_index_spec.rb @@ -63,10 +63,6 @@ @array.bsearch_index { |x| -1 }.should be_nil end - it "returns the middle element when block always returns zero" do - @array.bsearch_index { |x| 0 }.should == 2 - end - context "magnitude does not effect the result" do it "returns the index of any matched elements where element is between 4n <= xn < 8n" do [1, 2].should include(@array.bsearch_index { |x| (1 - x / 4) * (2**100) }) diff --git a/spec/ruby/core/dir/glob_spec.rb b/spec/ruby/core/dir/glob_spec.rb index 61115b5eec2..32f515c81d8 100644 --- a/spec/ruby/core/dir/glob_spec.rb +++ b/spec/ruby/core/dir/glob_spec.rb @@ -106,11 +106,11 @@ ruby_version_is '3.1' do it "recursively matches files and directories in nested dot subdirectory except . with 'nested/**/*' from the current directory and option File::FNM_DOTMATCH" do expected = %w[ - nested/. - nested/.dotsubir - nested/.dotsubir/.dotfile - nested/.dotsubir/nondotfile - ] + nested/. + nested/.dotsubir + nested/.dotsubir/.dotfile + nested/.dotsubir/nondotfile + ] Dir.glob('nested/**/*', File::FNM_DOTMATCH).sort.should == expected.sort end diff --git a/spec/ruby/core/dir/home_spec.rb b/spec/ruby/core/dir/home_spec.rb index 90a008faf12..3cf745ab468 100644 --- a/spec/ruby/core/dir/home_spec.rb +++ b/spec/ruby/core/dir/home_spec.rb @@ -40,22 +40,21 @@ home.should == "C:/rubyspäc/home" home.encoding.should == Encoding::UTF_8 end - end - it "retrieves the directory from HOME, USERPROFILE, HOMEDRIVE/HOMEPATH and the WinAPI in that order" do - old_dirs = [ENV.delete('HOME'), ENV.delete('USERPROFILE'), ENV.delete('HOMEDRIVE'), ENV.delete('HOMEPATH')] + it "retrieves the directory from HOME, USERPROFILE, HOMEDRIVE/HOMEPATH and the WinAPI in that order" do + old_dirs = [ENV.delete('HOME'), ENV.delete('USERPROFILE'), ENV.delete('HOMEDRIVE'), ENV.delete('HOMEPATH')] - Dir.home.should == old_dirs[1].gsub("\\", "/") - ENV['HOMEDRIVE'] = "C:" - ENV['HOMEPATH'] = "\\rubyspec\\home1" - Dir.home.should == "C:/rubyspec/home1" - ENV['USERPROFILE'] = "C:\\rubyspec\\home2" - # https://bugs.ruby-lang.org/issues/19244 - # Dir.home.should == "C:/rubyspec/home2" - ENV['HOME'] = "C:\\rubyspec\\home3" - Dir.home.should == "C:/rubyspec/home3" - ensure - ENV['HOME'], ENV['USERPROFILE'], ENV['HOMEDRIVE'], ENV['HOMEPATH'] = *old_dirs + Dir.home.should == old_dirs[1].gsub("\\", "/") + ENV['HOMEDRIVE'] = "C:" + ENV['HOMEPATH'] = "\\rubyspec\\home1" + Dir.home.should == "C:/rubyspec/home1" + ENV['USERPROFILE'] = "C:\\rubyspec\\home2" + Dir.home.should == "C:/rubyspec/home2" + ENV['HOME'] = "C:\\rubyspec\\home3" + Dir.home.should == "C:/rubyspec/home3" + ensure + ENV['HOME'], ENV['USERPROFILE'], ENV['HOMEDRIVE'], ENV['HOMEPATH'] = *old_dirs + end end end end diff --git a/spec/ruby/core/exception/detailed_message_spec.rb b/spec/ruby/core/exception/detailed_message_spec.rb index bd85927dbe6..fbe4443daab 100644 --- a/spec/ruby/core/exception/detailed_message_spec.rb +++ b/spec/ruby/core/exception/detailed_message_spec.rb @@ -7,6 +7,14 @@ RuntimeError.new("new error").detailed_message.should == "new error (RuntimeError)" end + it "is called by #full_message to allow message customization" do + exception = Exception.new("new error") + def exception.detailed_message(**) + "#{message}" + end + exception.full_message(highlight: false).should.include? "new error" + end + it "accepts highlight keyword argument and adds escape control sequences" do RuntimeError.new("new error").detailed_message(highlight: true).should == "\e[1mnew error (\e[1;4mRuntimeError\e[m\e[1m)\e[m" end @@ -23,13 +31,13 @@ RuntimeError.new("").detailed_message.should == "unhandled exception" end - it "returns just class name for an instance of RuntimeError sublass with empty message" do + it "returns just class name for an instance of RuntimeError subclass with empty message" do DetailedMessageSpec::C.new("").detailed_message.should == "DetailedMessageSpec::C" end it "returns a generated class name for an instance of RuntimeError anonymous subclass with empty message" do klass = Class.new(RuntimeError) - klass.new("").detailed_message.should =~ /\A#\z/ + klass.new("").detailed_message.should =~ /\A#\z/ end end end diff --git a/spec/ruby/core/exception/full_message_spec.rb b/spec/ruby/core/exception/full_message_spec.rb index ee66582022e..e15649ca756 100644 --- a/spec/ruby/core/exception/full_message_spec.rb +++ b/spec/ruby/core/exception/full_message_spec.rb @@ -107,21 +107,49 @@ ruby_version_is "3.2" do it "relies on #detailed_message" do e = RuntimeError.new("new error") - e.define_singleton_method(:detailed_message) { |**opt| "DETAILED MESSAGE" } + e.define_singleton_method(:detailed_message) { |**| "DETAILED MESSAGE" } e.full_message.lines.first.should =~ /DETAILED MESSAGE/ end - it "passes all its own keyword arguments to #detailed_message" do + it "passes all its own keyword arguments (with :highlight default value and without :order default value) to #detailed_message" do e = RuntimeError.new("new error") - opt_ = nil - e.define_singleton_method(:detailed_message) do |**opt| - opt_ = opt + options_passed = nil + e.define_singleton_method(:detailed_message) do |**options| + options_passed = options "DETAILED MESSAGE" end e.full_message(foo: "bar") - opt_.should == { foo: "bar", highlight: Exception.to_tty? } + options_passed.should == { foo: "bar", highlight: Exception.to_tty? } + end + + it "converts #detailed_message returned value to String if it isn't a String" do + message = Object.new + def message.to_str; "DETAILED MESSAGE"; end + + e = RuntimeError.new("new error") + e.define_singleton_method(:detailed_message) { |**| message } + + e.full_message.lines.first.should =~ /DETAILED MESSAGE/ + end + + it "uses class name if #detailed_message returns nil" do + e = RuntimeError.new("new error") + e.define_singleton_method(:detailed_message) { |**| nil } + + e.full_message(highlight: false).lines.first.should =~ /RuntimeError/ + e.full_message(highlight: true).lines.first.should =~ /#{Regexp.escape("\e[1;4mRuntimeError\e[m")}/ + end + + it "uses class name if exception object doesn't respond to #detailed_message" do + e = RuntimeError.new("new error") + class << e + undef :detailed_message + end + + e.full_message(highlight: false).lines.first.should =~ /RuntimeError/ + e.full_message(highlight: true).lines.first.should =~ /#{Regexp.escape("\e[1;4mRuntimeError\e[m")}/ end end end diff --git a/spec/ruby/core/file/flock_spec.rb b/spec/ruby/core/file/flock_spec.rb index 751e99d994e..070d830bc47 100644 --- a/spec/ruby/core/file/flock_spec.rb +++ b/spec/ruby/core/file/flock_spec.rb @@ -30,7 +30,7 @@ it "returns false if trying to lock an exclusively locked file" do @file.flock File::LOCK_EX - ruby_exe(<<-END_OF_CODE, escape: true).should == "false" + ruby_exe(<<-END_OF_CODE).should == "false" File.open('#{@name}', "w") do |f2| print f2.flock(File::LOCK_EX | File::LOCK_NB).to_s end @@ -40,7 +40,7 @@ it "blocks if trying to lock an exclusively locked file" do @file.flock File::LOCK_EX - out = ruby_exe(<<-END_OF_CODE, escape: true) + out = ruby_exe(<<-END_OF_CODE) running = false t = Thread.new do diff --git a/spec/ruby/core/file/new_spec.rb b/spec/ruby/core/file/new_spec.rb index 715ac1aaf33..3e2641aed3e 100644 --- a/spec/ruby/core/file/new_spec.rb +++ b/spec/ruby/core/file/new_spec.rb @@ -188,6 +188,12 @@ }.should raise_error(Errno::EEXIST, /File exists/) end + it "does not use the given block and warns to use File::open" do + -> { + @fh = File.new(@file) { raise } + }.should complain(/warning: File::new\(\) does not take block; use File::open\(\) instead/) + end + it "raises a TypeError if the first parameter can't be coerced to a string" do -> { File.new(true) }.should raise_error(TypeError) -> { File.new(false) }.should raise_error(TypeError) diff --git a/spec/ruby/core/file/shared/fnmatch.rb b/spec/ruby/core/file/shared/fnmatch.rb index f943c01eb8c..db4b5c5d8c6 100644 --- a/spec/ruby/core/file/shared/fnmatch.rb +++ b/spec/ruby/core/file/shared/fnmatch.rb @@ -102,6 +102,7 @@ it "matches ranges of characters using exclusive bracket expression (e.g. [^t] or [!t])" do File.send(@method, 'ca[^t]', 'cat').should == false + File.send(@method, 'ca[^t]', 'cas').should == true File.send(@method, 'ca[!t]', 'cat').should == false end @@ -172,9 +173,19 @@ File.should_not.send(@method, '*/*', 'dave/.profile', File::FNM_PATHNAME) end - it "matches patterns with leading periods to dotfiles by default" do + it "matches patterns with leading periods to dotfiles" do File.send(@method, '.*', '.profile').should == true + File.send(@method, '.*', '.profile', File::FNM_PATHNAME).should == true File.send(@method, ".*file", "nondotfile").should == false + File.send(@method, ".*file", "nondotfile", File::FNM_PATHNAME).should == false + end + + it "does not match directories with leading periods by default with FNM_PATHNAME" do + File.send(@method, '.*', '.directory/nondotfile', File::FNM_PATHNAME).should == false + File.send(@method, '.*', '.directory/.profile', File::FNM_PATHNAME).should == false + File.send(@method, '.*', 'foo/.directory/nondotfile', File::FNM_PATHNAME).should == false + File.send(@method, '.*', 'foo/.directory/.profile', File::FNM_PATHNAME).should == false + File.send(@method, '**/.dotfile', '.dotsubdir/.dotfile', File::FNM_PATHNAME).should == false end it "matches leading periods in filenames when flags includes FNM_DOTMATCH" do @@ -228,6 +239,33 @@ File.send(@method, pattern, 'a/.b/c/foo', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true end + it "has special handling for ./ when using * and FNM_PATHNAME" do + File.send(@method, './*', '.', File::FNM_PATHNAME).should be_false + File.send(@method, './*', './', File::FNM_PATHNAME).should be_true + File.send(@method, './*/', './', File::FNM_PATHNAME).should be_false + File.send(@method, './**', './', File::FNM_PATHNAME).should be_true + File.send(@method, './**/', './', File::FNM_PATHNAME).should be_true + File.send(@method, './*', '.', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_false + File.send(@method, './*', './', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true + File.send(@method, './*/', './', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_false + File.send(@method, './**', './', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true + File.send(@method, './**/', './', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true + end + + it "matches **/* with FNM_PATHNAME to recurse directories" do + File.send(@method, 'nested/**/*', 'nested/subdir', File::FNM_PATHNAME).should be_true + File.send(@method, 'nested/**/*', 'nested/subdir/file', File::FNM_PATHNAME).should be_true + File.send(@method, 'nested/**/*', 'nested/.dotsubdir', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true + File.send(@method, 'nested/**/*', 'nested/.dotsubir/.dotfile', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true + end + + it "matches ** with FNM_PATHNAME only in current directory" do + File.send(@method, 'nested/**', 'nested/subdir', File::FNM_PATHNAME).should be_true + File.send(@method, 'nested/**', 'nested/subdir/file', File::FNM_PATHNAME).should be_false + File.send(@method, 'nested/**', 'nested/.dotsubdir', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_true + File.send(@method, 'nested/**', 'nested/.dotsubir/.dotfile', File::FNM_PATHNAME | File::FNM_DOTMATCH).should be_false + end + it "accepts an object that has a #to_path method" do File.send(@method, '\*', mock_to_path('a')).should == false end diff --git a/spec/ruby/core/io/copy_stream_spec.rb b/spec/ruby/core/io/copy_stream_spec.rb index df9c5c7390e..ffa2ea992ce 100644 --- a/spec/ruby/core/io/copy_stream_spec.rb +++ b/spec/ruby/core/io/copy_stream_spec.rb @@ -69,9 +69,12 @@ end it "raises an IOError if the destination IO is not open for writing" do - @to_io.close - @to_io = new_io @to_name, "r" - -> { IO.copy_stream @object.from, @to_io }.should raise_error(IOError) + to_io = new_io __FILE__, "r" + begin + -> { IO.copy_stream @object.from, to_io }.should raise_error(IOError) + ensure + to_io.close + end end it "does not close the destination IO" do @@ -109,7 +112,8 @@ end after :each do - rm_r @to_name, @from_bigfile + rm_r @to_name if @to_name + rm_r @from_bigfile end describe "from an IO" do @@ -164,6 +168,25 @@ it_behaves_like :io_copy_stream_to_io, nil, IOSpecs::CopyStream it_behaves_like :io_copy_stream_to_io_with_offset, nil, IOSpecs::CopyStream end + + describe "to a Tempfile" do + before :all do + require 'tempfile' + end + + before :each do + @to_io = Tempfile.new("rubyspec_copy_stream", encoding: Encoding::BINARY, mode: File::RDONLY) + @to_name = @to_io.path + end + + after :each do + @to_io.close! + @to_name = nil # do not rm_r it, already done by Tempfile#close! + end + + it_behaves_like :io_copy_stream_to_io, nil, IOSpecs::CopyStream + it_behaves_like :io_copy_stream_to_io_with_offset, nil, IOSpecs::CopyStream + end end describe "from a file name" do @@ -277,10 +300,8 @@ @io.should_not_receive(:pos) IO.copy_stream(@io, @to_name) end - end - describe "with a destination that does partial reads" do before do @from_out, @from_in = IO.pipe diff --git a/spec/ruby/core/io/gets_spec.rb b/spec/ruby/core/io/gets_spec.rb index 73d76b3abdf..ca64bf860e4 100644 --- a/spec/ruby/core/io/gets_spec.rb +++ b/spec/ruby/core/io/gets_spec.rb @@ -24,6 +24,12 @@ end end + it "sets $_ to nil after the last line has been read" do + while @io.gets + end + $_.should be_nil + end + it "returns nil if called at the end of the stream" do IOSpecs.lines.length.times { @io.gets } @io.gets.should == nil diff --git a/spec/ruby/core/io/new_spec.rb b/spec/ruby/core/io/new_spec.rb index 9d14ec18ad4..979ac0efcbd 100644 --- a/spec/ruby/core/io/new_spec.rb +++ b/spec/ruby/core/io/new_spec.rb @@ -5,6 +5,12 @@ describe "IO.new" do it_behaves_like :io_new, :new + + it "does not use the given block and warns to use IO::open" do + -> { + @io = IO.send(@method, @fd) { raise } + }.should complain(/warning: IO::new\(\) does not take block; use IO::open\(\) instead/) + end end describe "IO.new" do diff --git a/spec/ruby/core/io/open_spec.rb b/spec/ruby/core/io/open_spec.rb index d3a3961df75..d151da9ce5a 100644 --- a/spec/ruby/core/io/open_spec.rb +++ b/spec/ruby/core/io/open_spec.rb @@ -37,6 +37,19 @@ ScratchPad.recorded.should == :called end + it "propagate an exception in the block after calling #close" do + -> do + IO.open(@fd, "w") do |io| + IOSpecs.io_mock(io, :close) do + super() + ScratchPad.record :called + end + raise Exception + end + end.should raise_error(Exception) + ScratchPad.recorded.should == :called + end + it "propagates an exception raised by #close that is not a StandardError" do -> do IO.open(@fd, "w") do |io| diff --git a/spec/ruby/core/io/pread_spec.rb b/spec/ruby/core/io/pread_spec.rb index fb0645dec68..aa496ee803a 100644 --- a/spec/ruby/core/io/pread_spec.rb +++ b/spec/ruby/core/io/pread_spec.rb @@ -26,11 +26,88 @@ buffer.should == "567" end + it "shrinks the buffer in case of less bytes read" do + buffer = "foo" + @file.pread(1, 0, buffer) + buffer.should == "1" + end + + it "grows the buffer in case of more bytes read" do + buffer = "foo" + @file.pread(5, 0, buffer) + buffer.should == "12345" + end + it "does not advance the file pointer" do @file.pread(4, 0).should == "1234" @file.read.should == "1234567890" end + it "ignores the current offset" do + @file.pos = 3 + @file.pread(4, 0).should == "1234" + end + + it "returns an empty string for maxlen = 0" do + @file.pread(0, 4).should == "" + end + + it "ignores the offset for maxlen = 0, even if it is out of file bounds" do + @file.pread(0, 400).should == "" + end + + it "does not reset the buffer when reading with maxlen = 0" do + buffer = "foo" + @file.pread(0, 4, buffer) + buffer.should == "foo" + + @file.pread(0, 400, buffer) + buffer.should == "foo" + end + + it "converts maxlen to Integer using #to_int" do + maxlen = mock('maxlen') + maxlen.should_receive(:to_int).and_return(4) + @file.pread(maxlen, 0).should == "1234" + end + + it "converts offset to Integer using #to_int" do + offset = mock('offset') + offset.should_receive(:to_int).and_return(0) + @file.pread(4, offset).should == "1234" + end + + it "converts a buffer to String using to_str" do + buffer = mock('buffer') + buffer.should_receive(:to_str).at_least(1).and_return("foo") + @file.pread(4, 0, buffer) + buffer.should_not.is_a?(String) + buffer.to_str.should == "1234" + end + + it "raises TypeError if maxlen is not an Integer and cannot be coerced into Integer" do + maxlen = Object.new + -> { @file.pread(maxlen, 0) }.should raise_error(TypeError, 'no implicit conversion of Object into Integer') + end + + it "raises TypeError if offset is not an Integer and cannot be coerced into Integer" do + offset = Object.new + -> { @file.pread(4, offset) }.should raise_error(TypeError, 'no implicit conversion of Object into Integer') + end + + it "raises ArgumentError for negative values of maxlen" do + -> { @file.pread(-4, 0) }.should raise_error(ArgumentError, 'negative string size (or size too big)') + end + + it "raised Errno::EINVAL for negative values of offset" do + -> { @file.pread(4, -1) }.should raise_error(Errno::EINVAL, /Invalid argument/) + end + + it "raises TypeError if the buffer is not a String and cannot be coerced into String" do + buffer = Object.new + -> { @file.pread(4, 0, buffer) }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end + it "raises EOFError if end-of-file is reached" do -> { @file.pread(1, 10) }.should raise_error(EOFError) end diff --git a/spec/ruby/core/io/pwrite_spec.rb b/spec/ruby/core/io/pwrite_spec.rb index c10578a8eb1..00d40db28d7 100644 --- a/spec/ruby/core/io/pwrite_spec.rb +++ b/spec/ruby/core/io/pwrite_spec.rb @@ -28,16 +28,42 @@ @file.pread(6, 0).should == "foobar" end + it "calls #to_s on the object to be written" do + object = mock("to_s") + object.should_receive(:to_s).and_return("foo") + @file.pwrite(object, 0) + @file.pread(3, 0).should == "foo" + end + + it "calls #to_int on the offset" do + offset = mock("to_int") + offset.should_receive(:to_int).and_return(2) + @file.pwrite("foo", offset) + @file.pread(3, 2).should == "foo" + end + it "raises IOError when file is not open in write mode" do File.open(@fname, "r") do |file| - -> { file.pwrite("foo", 1) }.should raise_error(IOError) + -> { file.pwrite("foo", 1) }.should raise_error(IOError, "not opened for writing") end end it "raises IOError when file is closed" do file = File.open(@fname, "w+") file.close - -> { file.pwrite("foo", 1) }.should raise_error(IOError) + -> { file.pwrite("foo", 1) }.should raise_error(IOError, "closed stream") + end + + it "raises a NoMethodError if object does not respond to #to_s" do + -> { + @file.pwrite(BasicObject.new, 0) + }.should raise_error(NoMethodError, /undefined method `to_s'/) + end + + it "raises a TypeError if the offset cannot be converted to an Integer" do + -> { + @file.pwrite("foo", Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") end end end diff --git a/spec/ruby/core/io/read_spec.rb b/spec/ruby/core/io/read_spec.rb index bd22fb6d6ea..996f70bf209 100644 --- a/spec/ruby/core/io/read_spec.rb +++ b/spec/ruby/core/io/read_spec.rb @@ -113,6 +113,15 @@ IO.read(@fname, 1, 10).should == nil end + it "returns an empty string when reading zero bytes" do + IO.read(@fname, 0).should == '' + end + + it "returns a String in BINARY when passed a size" do + IO.read(@fname, 1).encoding.should == Encoding::BINARY + IO.read(@fname, 0).encoding.should == Encoding::BINARY + end + it "raises an Errno::ENOENT when the requested file does not exist" do rm_r @fname -> { IO.read @fname }.should raise_error(Errno::ENOENT) @@ -274,6 +283,14 @@ @io.read(4).should == '7890' end + it "treats first nil argument as no length limit" do + @io.read(nil).should == @contents + end + + it "raises an ArgumentError when not passed a valid length" do + -> { @io.read(-1) }.should raise_error(ArgumentError) + end + it "clears the output buffer if there is nothing to read" do @io.pos = 10 @@ -565,6 +582,7 @@ it "returns a String in BINARY when passed a size" do @io.read(4).encoding.should equal(Encoding::BINARY) + @io.read(0).encoding.should equal(Encoding::BINARY) end it "does not change the buffer's encoding when passed a limit" do diff --git a/spec/ruby/core/io/shared/each.rb b/spec/ruby/core/io/shared/each.rb index dbc0178dd63..aca622834f3 100644 --- a/spec/ruby/core/io/shared/each.rb +++ b/spec/ruby/core/io/shared/each.rb @@ -33,10 +33,6 @@ $_.should == "test" end - it "returns self" do - @io.send(@method) { |l| l }.should equal(@io) - end - it "raises an IOError when self is not readable" do -> { IOSpecs.closed_io.send(@method) {} }.should raise_error(IOError) end diff --git a/spec/ruby/core/kernel/exec_spec.rb b/spec/ruby/core/kernel/exec_spec.rb index 1b4a7ae6f48..3d9520ad67a 100644 --- a/spec/ruby/core/kernel/exec_spec.rb +++ b/spec/ruby/core/kernel/exec_spec.rb @@ -7,12 +7,12 @@ end it "runs the specified command, replacing current process" do - ruby_exe('exec "echo hello"; puts "fail"', escape: true).should == "hello\n" + ruby_exe('exec "echo hello"; puts "fail"').should == "hello\n" end end describe "Kernel.exec" do it "runs the specified command, replacing current process" do - ruby_exe('Kernel.exec "echo hello"; puts "fail"', escape: true).should == "hello\n" + ruby_exe('Kernel.exec "echo hello"; puts "fail"').should == "hello\n" end end diff --git a/spec/ruby/core/kernel/lambda_spec.rb b/spec/ruby/core/kernel/lambda_spec.rb index 2f7abc749cb..565536ac0d6 100644 --- a/spec/ruby/core/kernel/lambda_spec.rb +++ b/spec/ruby/core/kernel/lambda_spec.rb @@ -26,42 +26,44 @@ l.lambda?.should be_true end - it "creates a lambda-style Proc if given a literal block via Kernel.public_send" do - suppress_warning do - l = Kernel.public_send(:lambda) { 42 } - l.lambda?.should be_true + ruby_version_is ""..."3.3" do + it "creates a lambda-style Proc if given a literal block via Kernel.public_send" do + suppress_warning do + l = Kernel.public_send(:lambda) { 42 } + l.lambda?.should be_true + end end - end - it "returns the passed Proc if given an existing Proc" do - some_proc = proc {} - l = suppress_warning {lambda(&some_proc)} - l.should equal(some_proc) - l.lambda?.should be_false - end + it "returns the passed Proc if given an existing Proc" do + some_proc = proc {} + l = suppress_warning {lambda(&some_proc)} + l.should equal(some_proc) + l.lambda?.should be_false + end - it "creates a lambda-style Proc when called with zsuper" do - suppress_warning do - l = KernelSpecs::LambdaSpecs::ForwardBlockWithZSuper.new.lambda { 42 } - l.lambda?.should be_true - l.call.should == 42 + it "creates a lambda-style Proc when called with zsuper" do + suppress_warning do + l = KernelSpecs::LambdaSpecs::ForwardBlockWithZSuper.new.lambda { 42 } + l.lambda?.should be_true + l.call.should == 42 - lambda { l.call(:extra) }.should raise_error(ArgumentError) + lambda { l.call(:extra) }.should raise_error(ArgumentError) + end end - end - it "returns the passed Proc if given an existing Proc through super" do - some_proc = proc { } - l = KernelSpecs::LambdaSpecs::SuperAmpersand.new.lambda(&some_proc) - l.should equal(some_proc) - l.lambda?.should be_false - end + it "returns the passed Proc if given an existing Proc through super" do + some_proc = proc { } + l = KernelSpecs::LambdaSpecs::SuperAmpersand.new.lambda(&some_proc) + l.should equal(some_proc) + l.lambda?.should be_false + end - it "does not create lambda-style Procs when captured with #method" do - kernel_lambda = method(:lambda) - l = suppress_warning {kernel_lambda.call { 42 }} - l.lambda?.should be_false - l.call(:extra).should == 42 + it "does not create lambda-style Procs when captured with #method" do + kernel_lambda = method(:lambda) + l = suppress_warning {kernel_lambda.call { 42 }} + l.lambda?.should be_false + l.call(:extra).should == 42 + end end it "checks the arity of the call when no args are specified" do @@ -137,8 +139,16 @@ def ret end context "when called without a literal block" do - it "warns when proc isn't a lambda" do - -> { lambda(&proc{}) }.should complain("#{__FILE__}:#{__LINE__}: warning: lambda without a literal block is deprecated; use the proc without lambda instead\n") + ruby_version_is ""..."3.3" do + it "warns when proc isn't a lambda" do + -> { lambda(&proc{}) }.should complain("#{__FILE__}:#{__LINE__}: warning: lambda without a literal block is deprecated; use the proc without lambda instead\n") + end + end + + ruby_version_is "3.3" do + it "raises when proc isn't a lambda" do + -> { lambda(&proc{}) }.should raise_error(ArgumentError, /the lambda method requires a literal block/) + end end it "doesn't warn when proc is lambda" do diff --git a/spec/ruby/core/kernel/printf_spec.rb b/spec/ruby/core/kernel/printf_spec.rb index d8f93ce4293..61bf955c259 100644 --- a/spec/ruby/core/kernel/printf_spec.rb +++ b/spec/ruby/core/kernel/printf_spec.rb @@ -31,6 +31,13 @@ object.should_receive(:write).with("string") Kernel.printf(object, "%s", "string") end + + it "calls #to_str to convert the format object to a String" do + object = mock('format string') + object.should_receive(:to_str).and_return("to_str: %i") + $stdout.should_receive(:write).with("to_str: 42") + Kernel.printf($stdout, object, 42) + end end describe "Kernel.printf" do diff --git a/spec/ruby/core/kernel/shared/load.rb b/spec/ruby/core/kernel/shared/load.rb index 5c41c19bf6c..0fe2d5ce167 100644 --- a/spec/ruby/core/kernel/shared/load.rb +++ b/spec/ruby/core/kernel/shared/load.rb @@ -1,5 +1,6 @@ main = self +# The big difference is Kernel#load does not attempt to add an extension to the passed path, unlike Kernel#require describe :kernel_load, shared: true do before :each do CodeLoadingSpecs.spec_setup @@ -10,22 +11,31 @@ CodeLoadingSpecs.spec_cleanup end - it "loads a non-extensioned file as a Ruby source file" do - path = File.expand_path "load_fixture", CODE_LOADING_DIR - @object.load(path).should be_true - ScratchPad.recorded.should == [:no_ext] - end + describe "(path resolution)" do + # This behavior is specific to Kernel#load, it differs for Kernel#require + it "loads a non-extensioned file as a Ruby source file" do + path = File.expand_path "load_fixture", CODE_LOADING_DIR + @object.load(path).should be_true + ScratchPad.recorded.should == [:no_ext] + end - it "loads a non .rb extensioned file as a Ruby source file" do - path = File.expand_path "load_fixture.ext", CODE_LOADING_DIR - @object.load(path).should be_true - ScratchPad.recorded.should == [:no_rb_ext] - end + it "loads a non .rb extensioned file as a Ruby source file" do + path = File.expand_path "load_fixture.ext", CODE_LOADING_DIR + @object.load(path).should be_true + ScratchPad.recorded.should == [:no_rb_ext] + end - it "loads from the current working directory" do - Dir.chdir CODE_LOADING_DIR do - @object.load("load_fixture.rb").should be_true - ScratchPad.recorded.should == [:loaded] + it "loads from the current working directory" do + Dir.chdir CODE_LOADING_DIR do + @object.load("load_fixture.rb").should be_true + ScratchPad.recorded.should == [:loaded] + end + end + + # This behavior is specific to Kernel#load, it differs for Kernel#require + it "does not look for a c-extension file when passed a path without extension (when no .rb is present)" do + path = File.join CODE_LOADING_DIR, "a", "load_fixture" + -> { @object.send(@method, path) }.should raise_error(LoadError) end end diff --git a/spec/ruby/core/kernel/shared/require.rb b/spec/ruby/core/kernel/shared/require.rb index 61081b200c3..250813191b8 100644 --- a/spec/ruby/core/kernel/shared/require.rb +++ b/spec/ruby/core/kernel/shared/require.rb @@ -212,6 +212,34 @@ describe :kernel_require, shared: true do describe "(path resolution)" do + it "loads .rb file when passed absolute path without extension" do + path = File.expand_path "load_fixture", CODE_LOADING_DIR + @object.send(@method, path).should be_true + # This should _not_ be [:no_ext] + ScratchPad.recorded.should == [:loaded] + end + + platform_is :linux, :darwin do + it "loads c-extension file when passed absolute path without extension when no .rb is present" do + # the error message is specific to what dlerror() returns + path = File.join CODE_LOADING_DIR, "a", "load_fixture" + -> { @object.send(@method, path) }.should raise_error(Exception, /file too short|not a mach-o file/) + end + end + + platform_is :darwin do + it "loads .bundle file when passed absolute path with .so" do + # the error message is specific to what dlerror() returns + path = File.join CODE_LOADING_DIR, "a", "load_fixture.so" + -> { @object.send(@method, path) }.should raise_error(Exception, /load_fixture\.bundle.+(file too short|not a mach-o file)/) + end + end + + it "does not try an extra .rb if the path already ends in .rb" do + path = File.join CODE_LOADING_DIR, "d", "load_fixture.rb" + -> { @object.send(@method, path) }.should raise_error(LoadError) + end + # For reference see [ruby-core:24155] in which matz confirms this feature is # intentional for security reasons. it "does not load a bare filename unless the current working directory is in $LOAD_PATH" do diff --git a/spec/ruby/core/kernel/sleep_spec.rb b/spec/ruby/core/kernel/sleep_spec.rb index 44b417a92e8..0570629723b 100644 --- a/spec/ruby/core/kernel/sleep_spec.rb +++ b/spec/ruby/core/kernel/sleep_spec.rb @@ -1,5 +1,4 @@ require_relative '../../spec_helper' -require_relative 'fixtures/classes' describe "Kernel#sleep" do it "is a private method" do diff --git a/spec/ruby/core/kernel/sprintf_spec.rb b/spec/ruby/core/kernel/sprintf_spec.rb index 7adf71be76f..9ef7f86f16d 100644 --- a/spec/ruby/core/kernel/sprintf_spec.rb +++ b/spec/ruby/core/kernel/sprintf_spec.rb @@ -3,6 +3,14 @@ require_relative 'shared/sprintf' require_relative 'shared/sprintf_encoding' +describe :kernel_sprintf_to_str, shared: true do + it "calls #to_str to convert the format object to a String" do + obj = mock('format string') + obj.should_receive(:to_str).and_return("to_str: %i") + @method.call(obj, 42).should == "to_str: 42" + end +end + describe "Kernel#sprintf" do it_behaves_like :kernel_sprintf, -> format, *args { sprintf(format, *args) @@ -11,6 +19,10 @@ it_behaves_like :kernel_sprintf_encoding, -> format, *args { sprintf(format, *args) } + + it_behaves_like :kernel_sprintf_to_str, -> format, *args { + sprintf(format, *args) + } end describe "Kernel.sprintf" do @@ -21,4 +33,8 @@ it_behaves_like :kernel_sprintf_encoding, -> format, *args { Kernel.sprintf(format, *args) } + + it_behaves_like :kernel_sprintf_to_str, -> format, *args { + Kernel.sprintf(format, *args) + } end diff --git a/spec/ruby/core/matchdata/values_at_spec.rb b/spec/ruby/core/matchdata/values_at_spec.rb index 4fd0bfc42ab..535719a2ee3 100644 --- a/spec/ruby/core/matchdata/values_at_spec.rb +++ b/spec/ruby/core/matchdata/values_at_spec.rb @@ -1,6 +1,6 @@ require_relative '../../spec_helper' -describe "Struct#values_at" do +describe "MatchData#values_at" do # Should be synchronized with core/array/values_at_spec.rb and core/struct/values_at_spec.rb # # /(.)(.)(\d+)(\d)/.match("THX1138: The Movie").to_a # => ["HX1138", "H", "X", "113", "8"] @@ -34,7 +34,7 @@ end it "supports beginningless Range" do - /(.)(.)(\d+)(\d)/.match("THX1138: The Movie").values_at(0..2).should == ["HX1138", "H", "X"] + /(.)(.)(\d+)(\d)/.match("THX1138: The Movie").values_at(..2).should == ["HX1138", "H", "X"] end it "returns an empty Array when Range is empty" do diff --git a/spec/ruby/core/method/super_method_spec.rb b/spec/ruby/core/method/super_method_spec.rb index f9a18f38785..c63a7aaa0fc 100644 --- a/spec/ruby/core/method/super_method_spec.rb +++ b/spec/ruby/core/method/super_method_spec.rb @@ -55,12 +55,10 @@ def overridden; end end end - ruby_version_is "2.7.3" do - context "after aliasing an inherited method" do - it "returns the expected super_method" do - method = MethodSpecs::InheritedMethods::C.new.method(:meow) - method.super_method.owner.should == MethodSpecs::InheritedMethods::A - end + context "after aliasing an inherited method" do + it "returns the expected super_method" do + method = MethodSpecs::InheritedMethods::C.new.method(:meow) + method.super_method.owner.should == MethodSpecs::InheritedMethods::A end end end diff --git a/spec/ruby/core/module/define_method_spec.rb b/spec/ruby/core/module/define_method_spec.rb index d742ece6bc6..e04bb87ceb0 100644 --- a/spec/ruby/core/module/define_method_spec.rb +++ b/spec/ruby/core/module/define_method_spec.rb @@ -724,7 +724,7 @@ def foo; end end end -describe "Method#define_method when passed a Method object" do +describe "Module#define_method when passed a Method object" do before :each do @klass = Class.new do def m(a, b, *c) @@ -749,7 +749,7 @@ def m(a, b, *c) end end -describe "Method#define_method when passed an UnboundMethod object" do +describe "Module#define_method when passed an UnboundMethod object" do before :each do @klass = Class.new do def m(a, b, *c) @@ -774,7 +774,7 @@ def m(a, b, *c) end end -describe "Method#define_method when passed a Proc object" do +describe "Module#define_method when passed a Proc object" do describe "and a method is defined inside" do it "defines the nested method in the default definee where the Proc was created" do prc = nil @@ -799,7 +799,7 @@ def nested_method_in_proc_for_define_method end end -describe "Method#define_method when passed a block" do +describe "Module#define_method when passed a block" do describe "behaves exactly like a lambda" do it "for return" do Class.new do diff --git a/spec/ruby/core/proc/lambda_spec.rb b/spec/ruby/core/proc/lambda_spec.rb index b2d3f503502..5c3c38fc2a6 100644 --- a/spec/ruby/core/proc/lambda_spec.rb +++ b/spec/ruby/core/proc/lambda_spec.rb @@ -14,9 +14,11 @@ Proc.new {}.lambda?.should be_false end - it "is preserved when passing a Proc with & to the lambda keyword" do - suppress_warning {lambda(&->{})}.lambda?.should be_true - suppress_warning {lambda(&proc{})}.lambda?.should be_false + ruby_version_is ""..."3.3" do + it "is preserved when passing a Proc with & to the lambda keyword" do + suppress_warning {lambda(&->{})}.lambda?.should be_true + suppress_warning {lambda(&proc{})}.lambda?.should be_false + end end it "is preserved when passing a Proc with & to the proc keyword" do diff --git a/spec/ruby/core/process/constants_spec.rb b/spec/ruby/core/process/constants_spec.rb index 4130bb58a55..616c54b8e1b 100644 --- a/spec/ruby/core/process/constants_spec.rb +++ b/spec/ruby/core/process/constants_spec.rb @@ -56,12 +56,18 @@ end platform_is :netbsd, :freebsd do - it "Process::RLIMIT_SBSIZE" do + it "has the correct constant values on NetBSD and FreeBSD" do Process::RLIMIT_SBSIZE.should == 9 # FIXME: what's it equal? Process::RLIMIT_AS.should == 10 end end + platform_is :freebsd do + it "has the correct constant values on FreeBSD" do + Process::RLIMIT_NPTS.should == 11 + end + end + platform_is :windows do it "does not define RLIMIT constants" do %i[ diff --git a/spec/ruby/core/process/exec_spec.rb b/spec/ruby/core/process/exec_spec.rb index 2fa8b089750..0f371b39c81 100644 --- a/spec/ruby/core/process/exec_spec.rb +++ b/spec/ruby/core/process/exec_spec.rb @@ -34,16 +34,16 @@ end it "runs the specified command, replacing current process" do - ruby_exe('Process.exec "echo hello"; puts "fail"', escape: true).should == "hello\n" + ruby_exe('Process.exec "echo hello"; puts "fail"').should == "hello\n" end it "sets the current directory when given the :chdir option" do tmpdir = tmp("")[0..-2] platform_is_not :windows do - ruby_exe("Process.exec(\"pwd\", chdir: #{tmpdir.inspect})", escape: true).should == "#{tmpdir}\n" + ruby_exe("Process.exec(\"pwd\", chdir: #{tmpdir.inspect})").should == "#{tmpdir}\n" end platform_is :windows do - ruby_exe("Process.exec(\"cd\", chdir: #{tmpdir.inspect})", escape: true).tr('\\', '/').should == "#{tmpdir}\n" + ruby_exe("Process.exec(\"cd\", chdir: #{tmpdir.inspect})").tr('\\', '/').should == "#{tmpdir}\n" end end @@ -73,13 +73,13 @@ platform_is_not :windows do it "subjects the specified command to shell expansion" do result = Dir.chdir(@dir) do - ruby_exe('Process.exec "echo *"', escape: true) + ruby_exe('Process.exec "echo *"') end result.chomp.should == @name end it "creates an argument array with shell parsing semantics for whitespace" do - ruby_exe('Process.exec "echo a b c d"', escape: true).should == "a b c d\n" + ruby_exe('Process.exec "echo a b c d"').should == "a b c d\n" end end @@ -87,13 +87,13 @@ # There is no shell expansion on Windows it "does not subject the specified command to shell expansion on Windows" do result = Dir.chdir(@dir) do - ruby_exe('Process.exec "echo *"', escape: true) + ruby_exe('Process.exec "echo *"') end result.should == "*\n" end it "does not create an argument array with shell parsing semantics for whitespace on Windows" do - ruby_exe('Process.exec "echo a b c d"', escape: true).should == "a b c d\n" + ruby_exe('Process.exec "echo a b c d"').should == "a b c d\n" end end @@ -105,7 +105,7 @@ platform_is :windows do cmd = '"cmd.exe", "/C", "echo", "*"' end - ruby_exe("Process.exec #{cmd}", escape: true).should == "*\n" + ruby_exe("Process.exec #{cmd}").should == "*\n" end end @@ -124,29 +124,29 @@ end it "sets environment variables in the child environment" do - ruby_exe('Process.exec({"FOO" => "BAR"}, "echo ' + var + '")', escape: true).should == "BAR\n" + ruby_exe('Process.exec({"FOO" => "BAR"}, "echo ' + var + '")').should == "BAR\n" end it "unsets environment variables whose value is nil" do platform_is_not :windows do - ruby_exe('Process.exec({"FOO" => nil}, "echo ' + var + '")', escape: true).should == "\n" + ruby_exe('Process.exec({"FOO" => nil}, "echo ' + var + '")').should == "\n" end platform_is :windows do # On Windows, echo-ing a non-existent env var is treated as echo-ing any other string of text - ruby_exe('Process.exec({"FOO" => nil}, "echo ' + var + '")', escape: true).should == var + "\n" + ruby_exe('Process.exec({"FOO" => nil}, "echo ' + var + '")').should == var + "\n" end end it "coerces environment argument using to_hash" do - ruby_exe('o = Object.new; def o.to_hash; {"FOO" => "BAR"}; end; Process.exec(o, "echo ' + var + '")', escape: true).should == "BAR\n" + ruby_exe('o = Object.new; def o.to_hash; {"FOO" => "BAR"}; end; Process.exec(o, "echo ' + var + '")').should == "BAR\n" end it "unsets other environment variables when given a true :unsetenv_others option" do platform_is_not :windows do - ruby_exe('Process.exec("echo ' + var + '", unsetenv_others: true)', escape: true).should == "\n" + ruby_exe('Process.exec("echo ' + var + '", unsetenv_others: true)').should == "\n" end platform_is :windows do - ruby_exe('Process.exec("' + ENV['COMSPEC'].gsub('\\', '\\\\\\') + ' /C echo ' + var + '", unsetenv_others: true)', escape: true).should == var + "\n" + ruby_exe('Process.exec("' + ENV['COMSPEC'].gsub('\\', '\\\\\\') + ' /C echo ' + var + '", unsetenv_others: true)').should == var + "\n" end end end @@ -154,19 +154,19 @@ describe "with a command array" do it "uses the first element as the command name and the second as the argv[0] value" do platform_is_not :windows do - ruby_exe('Process.exec(["/bin/sh", "argv_zero"], "-c", "echo $0")', escape: true).should == "argv_zero\n" + ruby_exe('Process.exec(["/bin/sh", "argv_zero"], "-c", "echo $0")').should == "argv_zero\n" end platform_is :windows do - ruby_exe('Process.exec(["cmd.exe", "/C"], "/C", "echo", "argv_zero")', escape: true).should == "argv_zero\n" + ruby_exe('Process.exec(["cmd.exe", "/C"], "/C", "echo", "argv_zero")').should == "argv_zero\n" end end it "coerces the argument using to_ary" do platform_is_not :windows do - ruby_exe('o = Object.new; def o.to_ary; ["/bin/sh", "argv_zero"]; end; Process.exec(o, "-c", "echo $0")', escape: true).should == "argv_zero\n" + ruby_exe('o = Object.new; def o.to_ary; ["/bin/sh", "argv_zero"]; end; Process.exec(o, "-c", "echo $0")').should == "argv_zero\n" end platform_is :windows do - ruby_exe('o = Object.new; def o.to_ary; ["cmd.exe", "/C"]; end; Process.exec(o, "/C", "echo", "argv_zero")', escape: true).should == "argv_zero\n" + ruby_exe('o = Object.new; def o.to_ary; ["cmd.exe", "/C"]; end; Process.exec(o, "/C", "echo", "argv_zero")').should == "argv_zero\n" end end @@ -200,7 +200,7 @@ end EOC - ruby_exe(cmd, escape: true) + ruby_exe(cmd) child_fd = IO.read(@child_fd_file).to_i child_fd.to_i.should > STDERR.fileno @@ -216,7 +216,7 @@ Process.exec("#{ruby_cmd(map_fd_fixture)} \#{f.fileno}", f.fileno => f.fileno) EOC - output = ruby_exe(cmd, escape: true) + output = ruby_exe(cmd) child_fd, close_on_exec = output.split child_fd.to_i.should > STDERR.fileno @@ -232,7 +232,7 @@ puts(f.close_on_exec?) EOC - output = ruby_exe(cmd, escape: true) + output = ruby_exe(cmd) output.split.should == ['true', 'false'] end end diff --git a/spec/ruby/core/process/times_spec.rb b/spec/ruby/core/process/times_spec.rb index d2610f64157..d3bff2cda9f 100644 --- a/spec/ruby/core/process/times_spec.rb +++ b/spec/ruby/core/process/times_spec.rb @@ -16,31 +16,4 @@ Process.times.utime.should > user end end - - # TODO: The precision of `getrusage` depends on platforms (OpenBSD - # seems not supporting under-milliseconds in fact); this example is - # very questionable as an example of Ruby, and it just repeats the - # guard condition. - guard -> do - 1000.times.any? do - # If getrusage has precision beyond milliseconds, there will be - # very likely at least one non-zero microsecond results when - # repeating enough. - time = Process.clock_gettime(:GETRUSAGE_BASED_CLOCK_PROCESS_CPUTIME_ID) - not ('%.6f' % time).end_with?('000') - end - rescue Errno::EINVAL - false - end do - it "uses getrusage when available to improve precision beyond milliseconds" do - max = 10_000 - - found = (max * 100).times.find do - time = Process.times.utime - !('%.6f' % time).end_with?('000') - end - - found.should_not == nil - end - end end diff --git a/spec/ruby/core/regexp/union_spec.rb b/spec/ruby/core/regexp/union_spec.rb index 80768364710..ea5a5053f72 100644 --- a/spec/ruby/core/regexp/union_spec.rb +++ b/spec/ruby/core/regexp/union_spec.rb @@ -43,6 +43,27 @@ Regexp.union("\u00A9".encode("ISO-8859-1"), "a".encode("UTF-8")).encoding.should == Encoding::ISO_8859_1 end + it "returns ASCII-8BIT if the regexp encodings are ASCII-8BIT and at least one has non-ASCII characters" do + us_ascii_implicit, us_ascii_explicit, binary = /abc/, /[\x00-\x7f]/n, /[\x80-\xBF]/n + us_ascii_implicit.encoding.should == Encoding::US_ASCII + us_ascii_explicit.encoding.should == Encoding::US_ASCII + binary.encoding.should == Encoding::BINARY + + Regexp.union(us_ascii_implicit, us_ascii_explicit, binary).encoding.should == Encoding::BINARY + Regexp.union(us_ascii_implicit, binary, us_ascii_explicit).encoding.should == Encoding::BINARY + Regexp.union(us_ascii_explicit, us_ascii_implicit, binary).encoding.should == Encoding::BINARY + Regexp.union(us_ascii_explicit, binary, us_ascii_implicit).encoding.should == Encoding::BINARY + Regexp.union(binary, us_ascii_implicit, us_ascii_explicit).encoding.should == Encoding::BINARY + Regexp.union(binary, us_ascii_explicit, us_ascii_implicit).encoding.should == Encoding::BINARY + end + + it "return US-ASCII if all patterns are ASCII-only" do + Regexp.union(/abc/e, /def/e).encoding.should == Encoding::US_ASCII + Regexp.union(/abc/n, /def/n).encoding.should == Encoding::US_ASCII + Regexp.union(/abc/s, /def/s).encoding.should == Encoding::US_ASCII + Regexp.union(/abc/u, /def/u).encoding.should == Encoding::US_ASCII + end + it "returns a Regexp with UTF-8 if one part is UTF-8" do Regexp.union(/probl[éeè]me/i, /help/i).encoding.should == Encoding::UTF_8 end @@ -54,83 +75,83 @@ it "raises ArgumentError if the arguments include conflicting ASCII-incompatible Strings" do -> { Regexp.union("a".encode("UTF-16LE"), "b".encode("UTF-16BE")) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-16LE and UTF-16BE') end it "raises ArgumentError if the arguments include conflicting ASCII-incompatible Regexps" do -> { Regexp.union(Regexp.new("a".encode("UTF-16LE")), Regexp.new("b".encode("UTF-16BE"))) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-16LE and UTF-16BE') end it "raises ArgumentError if the arguments include conflicting fixed encoding Regexps" do -> { Regexp.union(Regexp.new("a".encode("UTF-8"), Regexp::FIXEDENCODING), Regexp.new("b".encode("US-ASCII"), Regexp::FIXEDENCODING)) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-8 and US-ASCII') end it "raises ArgumentError if the arguments include a fixed encoding Regexp and a String containing non-ASCII-compatible characters in a different encoding" do -> { Regexp.union(Regexp.new("a".encode("UTF-8"), Regexp::FIXEDENCODING), "\u00A9".encode("ISO-8859-1")) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-8 and ISO-8859-1') end it "raises ArgumentError if the arguments include a String containing non-ASCII-compatible characters and a fixed encoding Regexp in a different encoding" do -> { Regexp.union("\u00A9".encode("ISO-8859-1"), Regexp.new("a".encode("UTF-8"), Regexp::FIXEDENCODING)) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: ISO-8859-1 and UTF-8') end it "raises ArgumentError if the arguments include an ASCII-incompatible String and an ASCII-only String" do -> { Regexp.union("a".encode("UTF-16LE"), "b".encode("UTF-8")) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, /ASCII incompatible encoding: UTF-16LE|incompatible encodings: UTF-16LE and US-ASCII/) end it "raises ArgumentError if the arguments include an ASCII-incompatible Regexp and an ASCII-only String" do -> { Regexp.union(Regexp.new("a".encode("UTF-16LE")), "b".encode("UTF-8")) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, /ASCII incompatible encoding: UTF-16LE|incompatible encodings: UTF-16LE and US-ASCII/) end it "raises ArgumentError if the arguments include an ASCII-incompatible String and an ASCII-only Regexp" do -> { Regexp.union("a".encode("UTF-16LE"), Regexp.new("b".encode("UTF-8"))) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, /ASCII incompatible encoding: UTF-16LE|incompatible encodings: UTF-16LE and US-ASCII/) end it "raises ArgumentError if the arguments include an ASCII-incompatible Regexp and an ASCII-only Regexp" do -> { Regexp.union(Regexp.new("a".encode("UTF-16LE")), Regexp.new("b".encode("UTF-8"))) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, /ASCII incompatible encoding: UTF-16LE|incompatible encodings: UTF-16LE and US-ASCII/) end it "raises ArgumentError if the arguments include an ASCII-incompatible String and a String containing non-ASCII-compatible characters in a different encoding" do -> { Regexp.union("a".encode("UTF-16LE"), "\u00A9".encode("ISO-8859-1")) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-16LE and ISO-8859-1') end it "raises ArgumentError if the arguments include an ASCII-incompatible Regexp and a String containing non-ASCII-compatible characters in a different encoding" do -> { Regexp.union(Regexp.new("a".encode("UTF-16LE")), "\u00A9".encode("ISO-8859-1")) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-16LE and ISO-8859-1') end it "raises ArgumentError if the arguments include an ASCII-incompatible String and a Regexp containing non-ASCII-compatible characters in a different encoding" do -> { Regexp.union("a".encode("UTF-16LE"), Regexp.new("\u00A9".encode("ISO-8859-1"))) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-16LE and ISO-8859-1') end it "raises ArgumentError if the arguments include an ASCII-incompatible Regexp and a Regexp containing non-ASCII-compatible characters in a different encoding" do -> { Regexp.union(Regexp.new("a".encode("UTF-16LE")), Regexp.new("\u00A9".encode("ISO-8859-1"))) - }.should raise_error(ArgumentError) + }.should raise_error(ArgumentError, 'incompatible encodings: UTF-16LE and ISO-8859-1') end it "uses to_str to convert arguments (if not Regexp)" do @@ -154,6 +175,8 @@ not_supported_on :opal do Regexp.union([/dogs/, /cats/i]).should == /(?-mix:dogs)|(?i-mx:cats)/ end - ->{Regexp.union(["skiing", "sledding"], [/dogs/, /cats/i])}.should raise_error(TypeError) + -> { + Regexp.union(["skiing", "sledding"], [/dogs/, /cats/i]) + }.should raise_error(TypeError, 'no implicit conversion of Array into String') end end diff --git a/spec/ruby/core/signal/signame_spec.rb b/spec/ruby/core/signal/signame_spec.rb index b66de9fc851..adfe895d97f 100644 --- a/spec/ruby/core/signal/signame_spec.rb +++ b/spec/ruby/core/signal/signame_spec.rb @@ -9,10 +9,22 @@ Signal.signame(-1).should == nil end + it "calls #to_int on an object to convert to an Integer" do + obj = mock('signal') + obj.should_receive(:to_int).and_return(0) + Signal.signame(obj).should == "EXIT" + end + it "raises a TypeError when the passed argument can't be coerced to Integer" do -> { Signal.signame("hello") }.should raise_error(TypeError) end + it "raises a TypeError when the passed argument responds to #to_int but does not return an Integer" do + obj = mock('signal') + obj.should_receive(:to_int).and_return('not an int') + -> { Signal.signame(obj) }.should raise_error(TypeError) + end + platform_is_not :windows do it "the original should take precedence over alias when looked up by number" do Signal.signame(Signal.list["ABRT"]).should == "ABRT" diff --git a/spec/ruby/core/signal/trap_spec.rb b/spec/ruby/core/signal/trap_spec.rb index 10e122e0721..b3186cda92b 100644 --- a/spec/ruby/core/signal/trap_spec.rb +++ b/spec/ruby/core/signal/trap_spec.rb @@ -221,6 +221,25 @@ Signal.trap(:HUP, @saved_trap).should equal(@proc) end + it "calls #to_str on an object to convert to a String" do + obj = mock("signal") + obj.should_receive(:to_str).exactly(2).times.and_return("HUP") + Signal.trap obj, @proc + Signal.trap(obj, @saved_trap).should equal(@proc) + end + + it "accepts Integer values" do + hup = Signal.list["HUP"] + Signal.trap hup, @proc + Signal.trap(hup, @saved_trap).should equal(@proc) + end + + it "does not call #to_int on an object to convert to an Integer" do + obj = mock("signal") + obj.should_not_receive(:to_int) + -> { Signal.trap obj, @proc }.should raise_error(ArgumentError, /bad signal type/) + end + it "raises ArgumentError when passed unknown signal" do -> { Signal.trap(300) { } }.should raise_error(ArgumentError, "invalid signal number (300)") -> { Signal.trap("USR10") { } }.should raise_error(ArgumentError, "unsupported signal `SIGUSR10'") diff --git a/spec/ruby/core/string/start_with_spec.rb b/spec/ruby/core/string/start_with_spec.rb index 81eed47f96e..35e33b46a66 100644 --- a/spec/ruby/core/string/start_with_spec.rb +++ b/spec/ruby/core/string/start_with_spec.rb @@ -11,7 +11,14 @@ "\xA9".should.start_with?("\xA9") # A9 is not a character head for UTF-8 end - ruby_bug "#19784", ""..."3.3" do + ruby_version_is ""..."3.3" do + it "does not check we are matching only part of a character" do + "\xe3\x81\x82".size.should == 1 + "\xe3\x81\x82".should.start_with?("\xe3") + end + end + + ruby_version_is "3.3" do # #19784 it "checks we are matching only part of a character" do "\xe3\x81\x82".size.should == 1 "\xe3\x81\x82".should_not.start_with?("\xe3") diff --git a/spec/ruby/core/thread/native_thread_id_spec.rb b/spec/ruby/core/thread/native_thread_id_spec.rb index 8460a1db8cf..17a08c8a150 100644 --- a/spec/ruby/core/thread/native_thread_id_spec.rb +++ b/spec/ruby/core/thread/native_thread_id_spec.rb @@ -19,8 +19,15 @@ main_thread_id = Thread.current.native_thread_id t_thread_id = t.native_thread_id - t_thread_id.should be_kind_of(Integer) + if ruby_version_is "3.3" + # native_thread_id can be nil on a M:N scheduler + t_thread_id.should be_kind_of(Integer) if t_thread_id != nil + else + t_thread_id.should be_kind_of(Integer) + end + main_thread_id.should_not == t_thread_id + t.run t.join t.native_thread_id.should == nil diff --git a/spec/ruby/core/unboundmethod/super_method_spec.rb b/spec/ruby/core/unboundmethod/super_method_spec.rb index 101c83b8b33..aa7c1293772 100644 --- a/spec/ruby/core/unboundmethod/super_method_spec.rb +++ b/spec/ruby/core/unboundmethod/super_method_spec.rb @@ -40,12 +40,10 @@ end end - ruby_version_is "2.7.3" do - context "after aliasing an inherited method" do - it "returns the expected super_method" do - method = MethodSpecs::InheritedMethods::C.instance_method(:meow) - method.super_method.owner.should == MethodSpecs::InheritedMethods::A - end + context "after aliasing an inherited method" do + it "returns the expected super_method" do + method = MethodSpecs::InheritedMethods::C.instance_method(:meow) + method.super_method.owner.should == MethodSpecs::InheritedMethods::A end end end diff --git a/spec/ruby/core/warning/element_reference_spec.rb b/spec/ruby/core/warning/element_reference_spec.rb index bd024d19b49..8cb4018c20b 100644 --- a/spec/ruby/core/warning/element_reference_spec.rb +++ b/spec/ruby/core/warning/element_reference_spec.rb @@ -1,11 +1,9 @@ require_relative '../../spec_helper' describe "Warning.[]" do - ruby_version_is '2.7.2' do - it "returns default values for categories :deprecated and :experimental" do - ruby_exe('p [Warning[:deprecated], Warning[:experimental]]').chomp.should == "[false, true]" - ruby_exe('p [Warning[:deprecated], Warning[:experimental]]', options: "-w").chomp.should == "[true, true]" - end + it "returns default values for categories :deprecated and :experimental" do + ruby_exe('p [Warning[:deprecated], Warning[:experimental]]').chomp.should == "[false, true]" + ruby_exe('p [Warning[:deprecated], Warning[:experimental]]', options: "-w").chomp.should == "[true, true]" end ruby_version_is '3.3' do diff --git a/spec/ruby/fixtures/code/d/load_fixture.rb.rb b/spec/ruby/fixtures/code/d/load_fixture.rb.rb new file mode 100644 index 00000000000..7e9217729a3 --- /dev/null +++ b/spec/ruby/fixtures/code/d/load_fixture.rb.rb @@ -0,0 +1 @@ +ScratchPad << :rbrb diff --git a/spec/ruby/language/alias_spec.rb b/spec/ruby/language/alias_spec.rb index ee928cbfaae..61fddb01849 100644 --- a/spec/ruby/language/alias_spec.rb +++ b/spec/ruby/language/alias_spec.rb @@ -252,7 +252,7 @@ def test_with_check(*args) it "on top level defines the alias on Object" do # because it defines on the default definee / current module - ruby_exe("def foo; end; alias bla foo; print method(:bla).owner", escape: true).should == "Object" + ruby_exe("def foo; end; alias bla foo; print method(:bla).owner").should == "Object" end it "raises a NameError when passed a missing name" do diff --git a/spec/ruby/language/case_spec.rb b/spec/ruby/language/case_spec.rb index 58e1aaed967..1a3925c9c6f 100644 --- a/spec/ruby/language/case_spec.rb +++ b/spec/ruby/language/case_spec.rb @@ -329,49 +329,6 @@ def bar; @calls << :bar; end 100 end.should == 100 end -end - -describe "The 'case'-construct with no target expression" do - it "evaluates the body of the first clause when at least one of its condition expressions is true" do - case - when true, false; 'foo' - end.should == 'foo' - end - - it "evaluates the body of the first when clause that is not false/nil" do - case - when false; 'foo' - when 2; 'bar' - when 1 == 1; 'baz' - end.should == 'bar' - - case - when false; 'foo' - when nil; 'foo' - when 1 == 1; 'bar' - end.should == 'bar' - end - - it "evaluates the body of the else clause if all when clauses are false/nil" do - case - when false; 'foo' - when nil; 'foo' - when 1 == 2; 'bar' - else 'baz' - end.should == 'baz' - end - - it "evaluates multiple conditional expressions as a boolean disjunction" do - case - when true, false; 'foo' - else 'bar' - end.should == 'foo' - - case - when false, true; 'foo' - else 'bar' - end.should == 'foo' - end it "evaluates true as only 'true' when true is the first clause" do case 1 @@ -442,6 +399,49 @@ def ===(o) :called end.should == :called end +end + +describe "The 'case'-construct with no target expression" do + it "evaluates the body of the first clause when at least one of its condition expressions is true" do + case + when true, false; 'foo' + end.should == 'foo' + end + + it "evaluates the body of the first when clause that is not false/nil" do + case + when false; 'foo' + when 2; 'bar' + when 1 == 1; 'baz' + end.should == 'bar' + + case + when false; 'foo' + when nil; 'foo' + when 1 == 1; 'bar' + end.should == 'bar' + end + + it "evaluates the body of the else clause if all when clauses are false/nil" do + case + when false; 'foo' + when nil; 'foo' + when 1 == 2; 'bar' + else 'baz' + end.should == 'baz' + end + + it "evaluates multiple conditional expressions as a boolean disjunction" do + case + when true, false; 'foo' + else 'bar' + end.should == 'foo' + + case + when false, true; 'foo' + else 'bar' + end.should == 'foo' + end # Homogeneous cases are often optimized to avoid === using a jump table, and should be tested separately. # See https://github.com/jruby/jruby/issues/6440 @@ -451,4 +451,13 @@ def ===(o) when 2; 'bar' end.should == 'foo' end + + it "expands arrays to lists of values" do + case + when *[false] + "foo" + when *[true] + "bar" + end.should == "bar" + end end diff --git a/spec/ruby/language/delegation_spec.rb b/spec/ruby/language/delegation_spec.rb index 3f24a79d5cb..020787aff60 100644 --- a/spec/ruby/language/delegation_spec.rb +++ b/spec/ruby/language/delegation_spec.rb @@ -38,28 +38,26 @@ def delegate(...) end end -ruby_version_is "2.7.3" do - describe "delegation with def(x, ...)" do - it "delegates rest and kwargs" do - a = Class.new(DelegationSpecs::Target) - a.class_eval(<<-RUBY) - def delegate(x, ...) - target(...) - end - RUBY +describe "delegation with def(x, ...)" do + it "delegates rest and kwargs" do + a = Class.new(DelegationSpecs::Target) + a.class_eval(<<-RUBY) + def delegate(x, ...) + target(...) + end + RUBY - a.new.delegate(0, 1, b: 2).should == [[1], {b: 2}] - end + a.new.delegate(0, 1, b: 2).should == [[1], {b: 2}] + end - it "delegates block" do - a = Class.new(DelegationSpecs::Target) - a.class_eval(<<-RUBY) - def delegate_block(x, ...) - target_block(...) - end - RUBY + it "delegates block" do + a = Class.new(DelegationSpecs::Target) + a.class_eval(<<-RUBY) + def delegate_block(x, ...) + target_block(...) + end + RUBY - a.new.delegate_block(0, 1, b: 2) { |x| x }.should == [{b: 2}, [1]] - end + a.new.delegate_block(0, 1, b: 2) { |x| x }.should == [{b: 2}, [1]] end end diff --git a/spec/ruby/library/bigdecimal/to_s_spec.rb b/spec/ruby/library/bigdecimal/to_s_spec.rb index 63dfaf5e7c3..ba9f960eb32 100644 --- a/spec/ruby/library/bigdecimal/to_s_spec.rb +++ b/spec/ruby/library/bigdecimal/to_s_spec.rb @@ -39,19 +39,17 @@ @bigneg.to_s("+").should_not =~ /^\+.*/ end - ruby_version_is ""..."3.3" do - it "inserts a space every n chars to fraction part, if integer n is supplied" do - re =\ - /\A0\.314 159 265 358 979 323 846 264 338 327 950 288 419 716 939 937E1\z/i - @bigdec.to_s(3).should =~ re - - str1 = '-123.45678 90123 45678 9' - BigDecimal("-123.45678901234567890").to_s('5F').should == str1 - # trailing zeroes removed - BigDecimal("1.00000000000").to_s('1F').should == "1.0" - # 0 is treated as no spaces - BigDecimal("1.2345").to_s('0F').should == "1.2345" - end + it "inserts a space every n chars to fraction part, if integer n is supplied" do + re =\ + /\A0\.314 159 265 358 979 323 846 264 338 327 950 288 419 716 939 937E1\z/i + @bigdec.to_s(3).should =~ re + + str1 = '-123.45678 90123 45678 9' + BigDecimal("-123.45678901234567890").to_s('5F').should == str1 + # trailing zeroes removed + BigDecimal("1.00000000000").to_s('1F').should == "1.0" + # 0 is treated as no spaces + BigDecimal("1.2345").to_s('0F').should == "1.2345" end version_is BigDecimal::VERSION, "3.1.5" do #ruby_version_is '3.3' do diff --git a/spec/ruby/library/cgi/escapeURIComponent_spec.rb b/spec/ruby/library/cgi/escapeURIComponent_spec.rb new file mode 100644 index 00000000000..2cf283c7786 --- /dev/null +++ b/spec/ruby/library/cgi/escapeURIComponent_spec.rb @@ -0,0 +1,57 @@ +require_relative '../../spec_helper' +require 'cgi' + +ruby_version_is "3.2" do + describe "CGI.escapeURIComponent" do + it "escapes whitespace" do + string = "&<>\" \xE3\x82\x86\xE3\x82\x93\xE3\x82\x86\xE3\x82\x93" + CGI.escapeURIComponent(string).should == '%26%3C%3E%22%20%E3%82%86%E3%82%93%E3%82%86%E3%82%93' + end + + it "does not escape with unreserved characters" do + string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + CGI.escapeURIComponent(string).should == "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + end + + it "supports String with invalid encoding" do + string = "\xC0\<\<".force_encoding("UTF-8") + CGI.escapeURIComponent(string).should == "%C0%3C%3C" + end + + it "processes String bytes one by one, not characters" do + CGI.escapeURIComponent("β").should == "%CE%B2" # "β" bytes representation is CE B2 + end + + it "raises a TypeError with nil" do + -> { + CGI.escapeURIComponent(nil) + }.should raise_error(TypeError, 'no implicit conversion of nil into String') + end + + it "encodes empty string" do + CGI.escapeURIComponent("").should == "" + end + + it "encodes single whitespace" do + CGI.escapeURIComponent(" ").should == "%20" + end + + it "encodes double whitespace" do + CGI.escapeURIComponent(" ").should == "%20%20" + end + + it "preserves encoding" do + string = "whatever".encode("ASCII-8BIT") + CGI.escapeURIComponent(string).encoding.should == Encoding::ASCII_8BIT + end + + it "uses implicit type conversion to String" do + object = Object.new + def object.to_str + "a b" + end + + CGI.escapeURIComponent(object).should == "a%20b" + end + end +end diff --git a/spec/ruby/library/cgi/initialize_spec.rb b/spec/ruby/library/cgi/initialize_spec.rb index f794f157f08..61bc971d491 100644 --- a/spec/ruby/library/cgi/initialize_spec.rb +++ b/spec/ruby/library/cgi/initialize_spec.rb @@ -29,8 +29,8 @@ it "does not extend self with any of the other HTML modules" do @cgi.send(:initialize) - @cgi.should_not be_kind_of(CGI::Html3) @cgi.should_not be_kind_of(CGI::HtmlExtension) + @cgi.should_not be_kind_of(CGI::Html3) @cgi.should_not be_kind_of(CGI::Html4) @cgi.should_not be_kind_of(CGI::Html4Tr) @cgi.should_not be_kind_of(CGI::Html4Fr) diff --git a/spec/ruby/library/datetime/rfc2822_spec.rb b/spec/ruby/library/datetime/rfc2822_spec.rb index 70bfca60b4b..83f7fa8d5b1 100644 --- a/spec/ruby/library/datetime/rfc2822_spec.rb +++ b/spec/ruby/library/datetime/rfc2822_spec.rb @@ -3,4 +3,8 @@ describe "DateTime.rfc2822" do it "needs to be reviewed for spec completeness" + + it "raises DateError if passed nil" do + -> { DateTime.rfc2822(nil) }.should raise_error(Date::Error, "invalid date") + end end diff --git a/spec/ruby/library/openssl/digest/append_spec.rb b/spec/ruby/library/openssl/digest/append_spec.rb new file mode 100644 index 00000000000..08802b72534 --- /dev/null +++ b/spec/ruby/library/openssl/digest/append_spec.rb @@ -0,0 +1,6 @@ +require_relative '../../../spec_helper' +require_relative 'shared/update' + +describe "OpenSSL::Digest#<<" do + it_behaves_like :openssl_digest_update, :<< +end diff --git a/spec/ruby/library/openssl/digest/block_length_spec.rb b/spec/ruby/library/openssl/digest/block_length_spec.rb new file mode 100644 index 00000000000..444ed9d20d8 --- /dev/null +++ b/spec/ruby/library/openssl/digest/block_length_spec.rb @@ -0,0 +1,44 @@ +require_relative '../../../spec_helper' +require_relative '../../../library/digest/sha1/shared/constants' +require_relative '../../../library/digest/sha256/shared/constants' +require_relative '../../../library/digest/sha384/shared/constants' +require_relative '../../../library/digest/sha512/shared/constants' +require 'openssl' + +describe "OpenSSL::Digest#block_length" do + context "when the digest object is created via a name argument" do + it "returns a SHA1 block length" do + OpenSSL::Digest.new('sha1').block_length.should == SHA1Constants::BlockLength + end + + it "returns a SHA256 block length" do + OpenSSL::Digest.new('sha256').block_length.should == SHA256Constants::BlockLength + end + + it "returns a SHA384 block length" do + OpenSSL::Digest.new('sha384').block_length.should == SHA384Constants::BlockLength + end + + it "returns a SHA512 block length" do + OpenSSL::Digest.new('sha512').block_length.should == SHA512Constants::BlockLength + end + end + + context "when the digest object is created via a subclass" do + it "returns a SHA1 block length" do + OpenSSL::Digest::SHA1.new.block_length.should == SHA1Constants::BlockLength + end + + it "returns a SHA256 block length" do + OpenSSL::Digest::SHA256.new.block_length.should == SHA256Constants::BlockLength + end + + it "returns a SHA384 block length" do + OpenSSL::Digest::SHA384.new.block_length.should == SHA384Constants::BlockLength + end + + it "returns a SHA512 block length" do + OpenSSL::Digest::SHA512.new.block_length.should == SHA512Constants::BlockLength + end + end +end diff --git a/spec/ruby/library/openssl/digest/digest_length_spec.rb b/spec/ruby/library/openssl/digest/digest_length_spec.rb new file mode 100644 index 00000000000..37d1cba9a7b --- /dev/null +++ b/spec/ruby/library/openssl/digest/digest_length_spec.rb @@ -0,0 +1,44 @@ +require_relative '../../../spec_helper' +require_relative '../../../library/digest/sha1/shared/constants' +require_relative '../../../library/digest/sha256/shared/constants' +require_relative '../../../library/digest/sha384/shared/constants' +require_relative '../../../library/digest/sha512/shared/constants' +require 'openssl' + +describe "OpenSSL::Digest#digest_length" do + context "when the digest object is created via a name argument" do + it "returns a SHA1 digest length" do + OpenSSL::Digest.new('sha1').digest_length.should == SHA1Constants::DigestLength + end + + it "returns a SHA256 digest length" do + OpenSSL::Digest.new('sha256').digest_length.should == SHA256Constants::DigestLength + end + + it "returns a SHA384 digest length" do + OpenSSL::Digest.new('sha384').digest_length.should == SHA384Constants::DigestLength + end + + it "returns a SHA512 digest length" do + OpenSSL::Digest.new('sha512').digest_length.should == SHA512Constants::DigestLength + end + end + + context "when the digest object is created via a subclass" do + it "returns a SHA1 digest length" do + OpenSSL::Digest::SHA1.new.digest_length.should == SHA1Constants::DigestLength + end + + it "returns a SHA256 digest length" do + OpenSSL::Digest::SHA256.new.digest_length.should == SHA256Constants::DigestLength + end + + it "returns a SHA384 digest length" do + OpenSSL::Digest::SHA384.new.digest_length.should == SHA384Constants::DigestLength + end + + it "returns a SHA512 digest length" do + OpenSSL::Digest::SHA512.new.digest_length.should == SHA512Constants::DigestLength + end + end +end diff --git a/spec/ruby/library/openssl/digest_spec.rb b/spec/ruby/library/openssl/digest/digest_spec.rb similarity index 84% rename from spec/ruby/library/openssl/digest_spec.rb rename to spec/ruby/library/openssl/digest/digest_spec.rb index b8e82d073f2..cf27d01b6d9 100644 --- a/spec/ruby/library/openssl/digest_spec.rb +++ b/spec/ruby/library/openssl/digest/digest_spec.rb @@ -1,12 +1,11 @@ -require_relative '../../spec_helper' -require_relative '../../library/digest/sha1/shared/constants' -require_relative '../../library/digest/sha256/shared/constants' -require_relative '../../library/digest/sha384/shared/constants' -require_relative '../../library/digest/sha512/shared/constants' +require_relative '../../../spec_helper' +require_relative '../../../library/digest/sha1/shared/constants' +require_relative '../../../library/digest/sha256/shared/constants' +require_relative '../../../library/digest/sha384/shared/constants' +require_relative '../../../library/digest/sha512/shared/constants' require 'openssl' -describe "OpenSSL::Digest" do - +describe "OpenSSL::Digest class methods" do describe ".digest" do it "returns a SHA1 digest" do OpenSSL::Digest.digest('sha1', SHA1Constants::Contents).should == SHA1Constants::Digest diff --git a/spec/ruby/library/openssl/digest/initialize_spec.rb b/spec/ruby/library/openssl/digest/initialize_spec.rb new file mode 100644 index 00000000000..1cd0409c4d8 --- /dev/null +++ b/spec/ruby/library/openssl/digest/initialize_spec.rb @@ -0,0 +1,141 @@ +require_relative '../../../spec_helper' +require_relative '../../../library/digest/sha1/shared/constants' +require_relative '../../../library/digest/sha256/shared/constants' +require_relative '../../../library/digest/sha384/shared/constants' +require_relative '../../../library/digest/sha512/shared/constants' +require 'openssl' + +describe "OpenSSL::Digest#initialize" do + describe "can be called with a digest name" do + it "returns a SHA1 object" do + OpenSSL::Digest.new("sha1").name.should == "SHA1" + end + + it "returns a SHA256 object" do + OpenSSL::Digest.new("sha256").name.should == "SHA256" + end + + it "returns a SHA384 object" do + OpenSSL::Digest.new("sha384").name.should == "SHA384" + end + + it "returns a SHA512 object" do + OpenSSL::Digest.new("sha512").name.should == "SHA512" + end + + it "throws an error when called with an unknown digest" do + -> { OpenSSL::Digest.new("wd40") }.should raise_error(RuntimeError, /Unsupported digest algorithm \(wd40\)/) + end + + it "cannot be called with a symbol" do + -> { OpenSSL::Digest.new(:SHA1) }.should raise_error(TypeError, /wrong argument type Symbol/) + end + + it "does not call #to_str on the argument" do + name = mock("digest name") + name.should_not_receive(:to_str) + -> { OpenSSL::Digest.new(name) }.should raise_error(TypeError, /wrong argument type/) + end + end + + describe "can be called with a digest object" do + it "returns a SHA1 object" do + OpenSSL::Digest.new(OpenSSL::Digest::SHA1.new).name.should == "SHA1" + end + + it "returns a SHA256 object" do + OpenSSL::Digest.new(OpenSSL::Digest::SHA256.new).name.should == "SHA256" + end + + it "returns a SHA384 object" do + OpenSSL::Digest.new(OpenSSL::Digest::SHA384.new).name.should == "SHA384" + end + + it "returns a SHA512 object" do + OpenSSL::Digest.new(OpenSSL::Digest::SHA512.new).name.should == "SHA512" + end + + it "ignores the state of the digest object" do + sha1 = OpenSSL::Digest.new('sha1', SHA1Constants::Contents) + OpenSSL::Digest.new(sha1).digest.should == SHA1Constants::BlankDigest + end + end + + it "cannot be called with a digest class" do + -> { OpenSSL::Digest.new(OpenSSL::Digest::SHA1) }.should raise_error(TypeError, /wrong argument type Class/) + end + + context "when called without an initial String argument" do + it "returns a SHA1 digest" do + OpenSSL::Digest.new("sha1").digest.should == SHA1Constants::BlankDigest + end + + it "returns a SHA256 digest" do + OpenSSL::Digest.new("sha256").digest.should == SHA256Constants::BlankDigest + end + + it "returns a SHA384 digest" do + OpenSSL::Digest.new("sha384").digest.should == SHA384Constants::BlankDigest + end + + it "returns a SHA512 digest" do + OpenSSL::Digest.new("sha512").digest.should == SHA512Constants::BlankDigest + end + end + + context "when called with an initial String argument" do + it "returns a SHA1 digest of that argument" do + OpenSSL::Digest.new("sha1", SHA1Constants::Contents).digest.should == SHA1Constants::Digest + end + + it "returns a SHA256 digest of that argument" do + OpenSSL::Digest.new("sha256", SHA256Constants::Contents).digest.should == SHA256Constants::Digest + end + + it "returns a SHA384 digest of that argument" do + OpenSSL::Digest.new("sha384", SHA384Constants::Contents).digest.should == SHA384Constants::Digest + end + + it "returns a SHA512 digest of that argument" do + OpenSSL::Digest.new("sha512", SHA512Constants::Contents).digest.should == SHA512Constants::Digest + end + end + + context "can be called on subclasses" do + describe "can be called without an initial String argument on subclasses" do + it "returns a SHA1 digest" do + OpenSSL::Digest::SHA1.new.digest.should == SHA1Constants::BlankDigest + end + + it "returns a SHA256 digest" do + OpenSSL::Digest::SHA256.new.digest.should == SHA256Constants::BlankDigest + end + + it "returns a SHA384 digest" do + OpenSSL::Digest::SHA384.new.digest.should == SHA384Constants::BlankDigest + end + + it "returns a SHA512 digest" do + OpenSSL::Digest::SHA512.new.digest.should == SHA512Constants::BlankDigest + end + end + + describe "can be called with an initial String argument on subclasses" do + it "returns a SHA1 digest" do + OpenSSL::Digest::SHA1.new(SHA1Constants::Contents).digest.should == SHA1Constants::Digest + end + + it "returns a SHA256 digest" do + OpenSSL::Digest::SHA256.new(SHA256Constants::Contents).digest.should == SHA256Constants::Digest + end + + it "returns a SHA384 digest" do + OpenSSL::Digest::SHA384.new(SHA384Constants::Contents).digest.should == SHA384Constants::Digest + end + + it "returns a SHA512 digest" do + OpenSSL::Digest::SHA512.new(SHA512Constants::Contents).digest.should == SHA512Constants::Digest + end + end + end +end diff --git a/spec/ruby/library/openssl/digest/name_spec.rb b/spec/ruby/library/openssl/digest/name_spec.rb new file mode 100644 index 00000000000..b379f35c1c1 --- /dev/null +++ b/spec/ruby/library/openssl/digest/name_spec.rb @@ -0,0 +1,16 @@ +require_relative '../../../spec_helper' +require 'openssl' + +describe "OpenSSL::Digest#name" do + it "returns the name of digest" do + OpenSSL::Digest.new('SHA1').name.should == 'SHA1' + end + + it "converts the name to the internal representation of OpenSSL" do + OpenSSL::Digest.new('sha1').name.should == 'SHA1' + end + + it "works on subclasses too" do + OpenSSL::Digest::SHA1.new.name.should == 'SHA1' + end +end diff --git a/spec/ruby/library/openssl/digest/reset_spec.rb b/spec/ruby/library/openssl/digest/reset_spec.rb new file mode 100644 index 00000000000..c19bf466330 --- /dev/null +++ b/spec/ruby/library/openssl/digest/reset_spec.rb @@ -0,0 +1,36 @@ +require_relative '../../../spec_helper' +require_relative '../../../library/digest/sha1/shared/constants' +require_relative '../../../library/digest/sha256/shared/constants' +require_relative '../../../library/digest/sha384/shared/constants' +require_relative '../../../library/digest/sha512/shared/constants' +require 'openssl' + +describe "OpenSSL::Digest#reset" do + it "works for a SHA1 digest" do + digest = OpenSSL::Digest.new('sha1', SHA1Constants::Contents) + digest.reset + digest.update(SHA1Constants::Contents) + digest.digest.should == SHA1Constants::Digest + end + + it "works for a SHA256 digest" do + digest = OpenSSL::Digest.new('sha256', SHA256Constants::Contents) + digest.reset + digest.update(SHA256Constants::Contents) + digest.digest.should == SHA256Constants::Digest + end + + it "works for a SHA384 digest" do + digest = OpenSSL::Digest.new('sha384', SHA384Constants::Contents) + digest.reset + digest.update(SHA384Constants::Contents) + digest.digest.should == SHA384Constants::Digest + end + + it "works for a SHA512 digest" do + digest = OpenSSL::Digest.new('sha512', SHA512Constants::Contents) + digest.reset + digest.update(SHA512Constants::Contents) + digest.digest.should == SHA512Constants::Digest + end +end diff --git a/spec/ruby/library/openssl/digest/shared/update.rb b/spec/ruby/library/openssl/digest/shared/update.rb new file mode 100644 index 00000000000..e5ff9dcb16f --- /dev/null +++ b/spec/ruby/library/openssl/digest/shared/update.rb @@ -0,0 +1,123 @@ +require_relative '../../../../library/digest/sha1/shared/constants' +require_relative '../../../../library/digest/sha256/shared/constants' +require_relative '../../../../library/digest/sha384/shared/constants' +require_relative '../../../../library/digest/sha512/shared/constants' +require 'openssl' + +describe :openssl_digest_update, shared: true do + context "when given input as a single string" do + it "returns a SHA1 digest" do + digest = OpenSSL::Digest.new('sha1') + digest.send(@method, SHA1Constants::Contents) + digest.digest.should == SHA1Constants::Digest + end + + it "returns a SHA256 digest" do + digest = OpenSSL::Digest.new('sha256') + digest.send(@method, SHA256Constants::Contents) + digest.digest.should == SHA256Constants::Digest + end + + it "returns a SHA384 digest" do + digest = OpenSSL::Digest.new('sha384') + digest.send(@method, SHA384Constants::Contents) + digest.digest.should == SHA384Constants::Digest + end + + it "returns a SHA512 digest" do + digest = OpenSSL::Digest.new('sha512') + digest.send(@method, SHA512Constants::Contents) + digest.digest.should == SHA512Constants::Digest + end + end + + context "when given input as multiple smaller substrings" do + it "returns a SHA1 digest" do + digest = OpenSSL::Digest.new('sha1') + SHA1Constants::Contents.each_char { |b| digest.send(@method, b) } + digest.digest.should == SHA1Constants::Digest + end + + it "returns a SHA256 digest" do + digest = OpenSSL::Digest.new('sha256') + SHA256Constants::Contents.each_char { |b| digest.send(@method, b) } + digest.digest.should == SHA256Constants::Digest + end + + it "returns a SHA384 digest" do + digest = OpenSSL::Digest.new('sha384') + SHA384Constants::Contents.each_char { |b| digest.send(@method, b) } + digest.digest.should == SHA384Constants::Digest + end + + it "returns a SHA512 digest" do + digest = OpenSSL::Digest.new('sha512') + SHA512Constants::Contents.each_char { |b| digest.send(@method, b) } + digest.digest.should == SHA512Constants::Digest + end + end + + context "when input is not a String and responds to #to_str" do + it "returns a SHA1 digest" do + str = mock('str') + str.should_receive(:to_str).and_return(SHA1Constants::Contents) + digest = OpenSSL::Digest.new('sha1') + digest.send(@method, str) + digest.digest.should == SHA1Constants::Digest + end + + it "returns a SHA256 digest" do + str = mock('str') + str.should_receive(:to_str).and_return(SHA256Constants::Contents) + digest = OpenSSL::Digest.new('sha256') + digest.send(@method, str) + digest.digest.should == SHA256Constants::Digest + end + + it "returns a SHA384 digest" do + str = mock('str') + str.should_receive(:to_str).and_return(SHA384Constants::Contents) + digest = OpenSSL::Digest.new('sha384') + digest.send(@method, str) + digest.digest.should == SHA384Constants::Digest + end + + it "returns a SHA512 digest" do + str = mock('str') + str.should_receive(:to_str).and_return(SHA512Constants::Contents) + digest = OpenSSL::Digest.new('sha512') + digest.send(@method, str) + digest.digest.should == SHA512Constants::Digest + end + end + + context "when input is not a String and does not respond to #to_str" do + it "raises a TypeError with SHA1" do + digest = OpenSSL::Digest.new('sha1') + -> { + digest.send(@method, Object.new) + }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end + + it "raises a TypeError with SHA256" do + digest = OpenSSL::Digest.new('sha256') + -> { + digest.send(@method, Object.new) + }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end + + it "raises a TypeError with SHA384" do + digest = OpenSSL::Digest.new('sha384') + -> { + digest.send(@method, Object.new) + }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end + + it "raises a TypeError with SHA512" do + digest = OpenSSL::Digest.new('sha512') + -> { + digest.send(@method, Object.new) + }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end + end +end diff --git a/spec/ruby/library/openssl/digest/update_spec.rb b/spec/ruby/library/openssl/digest/update_spec.rb new file mode 100644 index 00000000000..3a90b06c6b1 --- /dev/null +++ b/spec/ruby/library/openssl/digest/update_spec.rb @@ -0,0 +1,6 @@ +require_relative '../../../spec_helper' +require_relative 'shared/update' + +describe "OpenSSL::Digest#update" do + it_behaves_like :openssl_digest_update, :update +end diff --git a/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb b/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb new file mode 100644 index 00000000000..ebf71482de1 --- /dev/null +++ b/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb @@ -0,0 +1,184 @@ +require_relative '../../../spec_helper' +require 'openssl' + +describe "OpenSSL::KDF.pbkdf2_hmac" do + before :each do + @defaults = { + salt: "\x00".b * 16, + iterations: 20_000, + length: 16, + hash: "sha1" + } + end + + it "creates the same value with the same input" do + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0".b + end + + it "supports nullbytes embedded in the password" do + key = OpenSSL::KDF.pbkdf2_hmac("sec\x00ret".b, **@defaults) + key.should == "\xB9\x7F\xB0\xC2\th\xC8<\x86\xF3\x94Ij7\xEF\xF1".b + end + + it "coerces the password into a String using #to_str" do + pass = mock("pass") + pass.should_receive(:to_str).and_return("secret") + key = OpenSSL::KDF.pbkdf2_hmac(pass, **@defaults) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0".b + end + + it "coerces the salt into a String using #to_str" do + salt = mock("salt") + salt.should_receive(:to_str).and_return("\x00".b * 16) + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, salt: salt) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0".b + end + + it "coerces the iterations into an Integer using #to_int" do + iterations = mock("iterations") + iterations.should_receive(:to_int).and_return(20_000) + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: iterations) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0".b + end + + it "coerces the length into an Integer using #to_int" do + length = mock("length") + length.should_receive(:to_int).and_return(16) + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, length: length) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0".b + end + + it "accepts a OpenSSL::Digest object as hash" do + hash = OpenSSL::Digest.new("sha1") + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, hash: hash) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0".b + end + + it "accepts an empty password" do + key = OpenSSL::KDF.pbkdf2_hmac("", **@defaults) + key.should == "k\x9F-\xB1\xF7\x9A\v\xA1(C\xF9\x85!P\xEF\x8C".b + end + + it "accepts an empty salt" do + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, salt: "") + key.should == "\xD5f\xE5\xEA\xF91\x1D\xD3evD\xED\xDB\xE80\x80".b + end + + it "accepts an empty length" do + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, length: 0) + key.should.empty? + end + + it "accepts an arbitrary length" do + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, length: 19) + key.should == "!\x99+\xF0^\xD0\x8BM\x158\xC4\xAC\x9C\xF1\xF0\xE0\xCF\xBB\x7F".b + end + + it "accepts any hash function known to OpenSSL" do + key = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, hash: "sha512") + key.should == "N\x12}D\xCE\x99\xDBC\x8E\xEC\xAAr\xEA1\xDF\xFF".b + end + + it "raises a TypeError when password is not a String and does not respond to #to_str" do + -> { + OpenSSL::KDF.pbkdf2_hmac(Object.new, **@defaults) + }.should raise_error(TypeError, "no implicit conversion of Object into String") + end + + it "raises a TypeError when salt is not a String and does not respond to #to_str" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, salt: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into String") + end + + it "raises a TypeError when iterations is not an Integer and does not respond to #to_int" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") + end + + it "raises a TypeError when length is not an Integer and does not respond to #to_int" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, length: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") + end + + it "raises a TypeError when hash is neither a String nor an OpenSSL::Digest" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, hash: Object.new) + }.should raise_error(TypeError, "wrong argument type Object (expected OpenSSL/Digest)") + end + + it "raises a TypeError when hash is neither a String nor an OpenSSL::Digest, it does not try to call #to_str" do + hash = mock("hash") + hash.should_not_receive(:to_str) + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, hash: hash) + }.should raise_error(TypeError, "wrong argument type MockObject (expected OpenSSL/Digest)") + end + + it "raises a RuntimeError for unknown digest algorithms" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, hash: "wd40") + }.should raise_error(RuntimeError, /Unsupported digest algorithm \(wd40\)/) + end + + it "treats salt as a required keyword" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults.except(:salt)) + }.should raise_error(ArgumentError, 'missing keyword: :salt') + end + + it "treats iterations as a required keyword" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults.except(:iterations)) + }.should raise_error(ArgumentError, 'missing keyword: :iterations') + end + + it "treats length as a required keyword" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults.except(:length)) + }.should raise_error(ArgumentError, 'missing keyword: :length') + end + + it "treats hash as a required keyword" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults.except(:hash)) + }.should raise_error(ArgumentError, 'missing keyword: :hash') + end + + it "treats all keywords as required" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret") + }.should raise_error(ArgumentError, 'missing keywords: :salt, :iterations, :length, :hash') + end + + + guard -> { OpenSSL::OPENSSL_VERSION_NUMBER < 0x30000000 } do + it "treats 0 or less iterations as a single iteration" do + salt = "\x00".b * 16 + length = 16 + hash = "sha1" + + # "Any iter less than 1 is treated as a single iteration." + key0 = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: 0) + key_negative = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: -1) + key1 = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: 1) + key0.should == key1 + key_negative.should == key1 + end + end + + guard -> { OpenSSL::OPENSSL_VERSION_NUMBER >= 0x30000000 } do + it "raises an OpenSSL::KDF::KDFError for 0 or less iterations" do + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: 0) + }.should raise_error(OpenSSL::KDF::KDFError, "PKCS5_PBKDF2_HMAC: invalid iteration count") + + -> { + OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: -1) + }.should raise_error(OpenSSL::KDF::KDFError, /PKCS5_PBKDF2_HMAC/) + end + end +end diff --git a/spec/ruby/library/openssl/kdf/scrypt_spec.rb b/spec/ruby/library/openssl/kdf/scrypt_spec.rb new file mode 100644 index 00000000000..d9b83eaa553 --- /dev/null +++ b/spec/ruby/library/openssl/kdf/scrypt_spec.rb @@ -0,0 +1,207 @@ +require_relative '../../../spec_helper' +require 'openssl' + +describe "OpenSSL::KDF.scrypt" do + before :each do + @defaults = { + salt: "\x00".b * 16, + N: 2**14, + r: 8, + p: 1, + length: 32 + } + end + + it "creates the same value with the same input" do + key = OpenSSL::KDF.scrypt("secret", **@defaults) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "supports nullbytes embedded into the password" do + key = OpenSSL::KDF.scrypt("sec\x00ret".b, **@defaults) + key.should == "\xF9\xA4\xA0\xF1p\xF4\xF0\xCAT\xB4v\xEB\r7\x88N\xF7\x15]Ns\xFCwt4a\xC9\xC6\xA7\x13\x81&".b + end + + it "coerces the password into a String using #to_str" do + pass = mock("pass") + pass.should_receive(:to_str).and_return("secret") + key = OpenSSL::KDF.scrypt(pass, **@defaults) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "coerces the salt into a String using #to_str" do + salt = mock("salt") + salt.should_receive(:to_str).and_return("\x00".b * 16) + key = OpenSSL::KDF.scrypt("secret", **@defaults, salt: salt) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "coerces the N into an Integer using #to_int" do + n = mock("N") + n.should_receive(:to_int).and_return(2**14) + key = OpenSSL::KDF.scrypt("secret", **@defaults, N: n) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "coerces the r into an Integer using #to_int" do + r = mock("r") + r.should_receive(:to_int).and_return(8) + key = OpenSSL::KDF.scrypt("secret", **@defaults, r: r) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "coerces the p into an Integer using #to_int" do + p = mock("p") + p.should_receive(:to_int).and_return(1) + key = OpenSSL::KDF.scrypt("secret", **@defaults, p: p) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "coerces the length into an Integer using #to_int" do + length = mock("length") + length.should_receive(:to_int).and_return(32) + key = OpenSSL::KDF.scrypt("secret", **@defaults, length: length) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D\x17}\xF2\x84T\xD4)\xC2>\xFE\x93\xE3\xF4".b + end + + it "accepts an empty password" do + key = OpenSSL::KDF.scrypt("", **@defaults) + key.should == "\xAA\xFC\xF5^E\x94v\xFFk\xE6\xF0vR\xE7\x13\xA7\xF5\x15'\x9A\xE4C\x9Dn\x18F_E\xD2\v\e\xB3".b + end + + it "accepts an empty salt" do + key = OpenSSL::KDF.scrypt("secret", **@defaults, salt: "") + key.should == "\x96\xACDl\xCB3/aN\xB0F\x8A#\xD7\x92\xD2O\x1E\v\xBB\xCE\xC0\xAA\xB9\x0F]\xB09\xEA8\xDD\e".b + end + + it "accepts a zero length" do + key = OpenSSL::KDF.scrypt("secret", **@defaults, length: 0) + key.should.empty? + end + + it "accepts an arbitrary length" do + key = OpenSSL::KDF.scrypt("secret", **@defaults, length: 19) + key.should == "h\xB2k\xDF]\xDA\xE1.-(\xCF\xAC\x91D\x8F\xC2a\x9C\x9D".b + end + + it "raises a TypeError when password is not a String and does not respond to #to_str" do + -> { + OpenSSL::KDF.scrypt(Object.new, **@defaults) + }.should raise_error(TypeError, "no implicit conversion of Object into String") + end + + it "raises a TypeError when salt is not a String and does not respond to #to_str" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, salt: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into String") + end + + it "raises a TypeError when N is not an Integer and does not respond to #to_int" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, N: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") + end + + it "raises a TypeError when r is not an Integer and does not respond to #to_int" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, r: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") + end + + it "raises a TypeError when p is not an Integer and does not respond to #to_int" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, p: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") + end + + it "raises a TypeError when length is not an Integer and does not respond to #to_int" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, length: Object.new) + }.should raise_error(TypeError, "no implicit conversion of Object into Integer") + end + + it "treats salt as a required keyword" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults.except(:salt)) + }.should raise_error(ArgumentError, 'missing keyword: :salt') + end + + it "treats N as a required keyword" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults.except(:N)) + }.should raise_error(ArgumentError, 'missing keyword: :N') + end + + it "treats r as a required keyword" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults.except(:r)) + }.should raise_error(ArgumentError, 'missing keyword: :r') + end + + it "treats p as a required keyword" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults.except(:p)) + }.should raise_error(ArgumentError, 'missing keyword: :p') + end + + it "treats length as a required keyword" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults.except(:length)) + }.should raise_error(ArgumentError, 'missing keyword: :length') + end + + it "treats all keywords as required" do + -> { + OpenSSL::KDF.scrypt("secret") + }.should raise_error(ArgumentError, 'missing keywords: :salt, :N, :r, :p, :length') + end + + it "requires N to be a power of 2" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, N: 2**14 - 1) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + end + + it "requires N to be at least 2" do + key = OpenSSL::KDF.scrypt("secret", **@defaults, N: 2) + key.should == "\x06A$a\xA9!\xBE\x01\x85\xA7\x18\xBCEa\x82\xC5\xFEl\x93\xAB\xBD\xF7\x8B\x84\v\xFC\eN\xEBQ\xE6\xD2".b + + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, N: 1) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, N: 0) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, N: -1) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + end + + it "requires r to be positive" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, r: 0) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, r: -1) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + end + + it "requires p to be positive" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, p: 0) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, p: -1) + }.should raise_error(OpenSSL::KDF::KDFError, /EVP_PBE_scrypt/) + end + + it "requires length to be not negative" do + -> { + OpenSSL::KDF.scrypt("secret", **@defaults, length: -1) + }.should raise_error(ArgumentError, "negative string size (or size too big)") + end +end diff --git a/spec/ruby/library/socket/ipsocket/getaddress_spec.rb b/spec/ruby/library/socket/ipsocket/getaddress_spec.rb index 746d2ab86bf..96324982e5c 100644 --- a/spec/ruby/library/socket/ipsocket/getaddress_spec.rb +++ b/spec/ruby/library/socket/ipsocket/getaddress_spec.rb @@ -19,7 +19,7 @@ # traditionally invalidly named ones. it "raises an error on unknown hostnames" do -> { - IPSocket.getaddress("rubyspecdoesntexist.fallingsnow.net") + IPSocket.getaddress("rubyspecdoesntexist.ruby-lang.org") }.should raise_error(SocketError) end end diff --git a/spec/ruby/library/socket/tcpserver/new_spec.rb b/spec/ruby/library/socket/tcpserver/new_spec.rb index 8d9696c9d86..dd1ba676bd9 100644 --- a/spec/ruby/library/socket/tcpserver/new_spec.rb +++ b/spec/ruby/library/socket/tcpserver/new_spec.rb @@ -97,6 +97,12 @@ addr[1].should be_kind_of(Integer) end + it "does not use the given block and warns to use TCPServer::open" do + -> { + @server = TCPServer.new(0) { raise } + }.should complain(/warning: TCPServer::new\(\) does not take block; use TCPServer::open\(\) instead/) + end + it "raises Errno::EADDRNOTAVAIL when the address is unknown" do -> { TCPServer.new("1.2.3.4", 0) }.should raise_error(Errno::EADDRNOTAVAIL) end diff --git a/spec/ruby/library/socket/tcpsocket/initialize_spec.rb b/spec/ruby/library/socket/tcpsocket/initialize_spec.rb index 065c8f4190a..3bec06c6974 100644 --- a/spec/ruby/library/socket/tcpsocket/initialize_spec.rb +++ b/spec/ruby/library/socket/tcpsocket/initialize_spec.rb @@ -4,6 +4,27 @@ describe 'TCPSocket#initialize' do it_behaves_like :tcpsocket_new, :new + + describe "with a running server" do + before :each do + @server = SocketSpecs::SpecTCPServer.new + @hostname = @server.hostname + end + + after :each do + if @socket + @socket.write "QUIT" + @socket.close + end + @server.shutdown + end + + it "does not use the given block and warns to use TCPSocket::open" do + -> { + @socket = TCPSocket.new(@hostname, @server.port, nil) { raise } + }.should complain(/warning: TCPSocket::new\(\) does not take block; use TCPSocket::open\(\) instead/) + end + end end describe 'TCPSocket#initialize' do diff --git a/spec/ruby/library/socket/tcpsocket/shared/new.rb b/spec/ruby/library/socket/tcpsocket/shared/new.rb index d60260a01b0..90f8d7e6a28 100644 --- a/spec/ruby/library/socket/tcpsocket/shared/new.rb +++ b/spec/ruby/library/socket/tcpsocket/shared/new.rb @@ -19,6 +19,9 @@ -> { TCPSocket.send(@method, "192.0.2.1", 80, connect_timeout: 0) }.should raise_error(Errno::ETIMEDOUT) + rescue Errno::ENETUNREACH + # In the case all network interfaces down. + # raise_error cannot deal with multiple expected exceptions end end @@ -27,6 +30,9 @@ -> { TCPSocket.send(@method, "192.0.2.1", 80, connect_timeout: 0) }.should raise_error(IO::TimeoutError) + rescue Errno::ENETUNREACH + # In the case all network interfaces down. + # raise_error cannot deal with multiple expected exceptions end end diff --git a/spec/ruby/library/socket/udpsocket/new_spec.rb b/spec/ruby/library/socket/udpsocket/new_spec.rb index 6cc0cadbcb4..79bfcb624d2 100644 --- a/spec/ruby/library/socket/udpsocket/new_spec.rb +++ b/spec/ruby/library/socket/udpsocket/new_spec.rb @@ -26,6 +26,12 @@ @socket.should be_an_instance_of(UDPSocket) end + it "does not use the given block and warns to use UDPSocket::open" do + -> { + @socket = UDPSocket.new { raise } + }.should complain(/warning: UDPSocket::new\(\) does not take block; use UDPSocket::open\(\) instead/) + end + it 'raises Errno::EAFNOSUPPORT or Errno::EPROTONOSUPPORT if unsupported family passed' do -> { UDPSocket.new(-1) }.should raise_error(SystemCallError) { |e| [Errno::EAFNOSUPPORT, Errno::EPROTONOSUPPORT].should include(e.class) diff --git a/spec/ruby/library/socket/unixserver/accept_nonblock_spec.rb b/spec/ruby/library/socket/unixserver/accept_nonblock_spec.rb index 30688b46b6d..dba3de7359d 100644 --- a/spec/ruby/library/socket/unixserver/accept_nonblock_spec.rb +++ b/spec/ruby/library/socket/unixserver/accept_nonblock_spec.rb @@ -1,9 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXServer#accept_nonblock" do - - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXServer#accept_nonblock" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -33,9 +32,7 @@ @server.accept_nonblock(exception: false).should == :wait_readable end end -end -with_feature :unix_socket do describe 'UNIXServer#accept_nonblock' do before do @path = SocketSpecs.socket_path diff --git a/spec/ruby/library/socket/unixserver/accept_spec.rb b/spec/ruby/library/socket/unixserver/accept_spec.rb index c05fbe7f226..624782d6b98 100644 --- a/spec/ruby/library/socket/unixserver/accept_spec.rb +++ b/spec/ruby/library/socket/unixserver/accept_spec.rb @@ -1,7 +1,7 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -platform_is_not :windows do +with_feature :unix_socket do describe "UNIXServer#accept" do before :each do @path = SocketSpecs.socket_path diff --git a/spec/ruby/library/socket/unixserver/for_fd_spec.rb b/spec/ruby/library/socket/unixserver/for_fd_spec.rb index 4f3816ad37b..e00c4d95267 100644 --- a/spec/ruby/library/socket/unixserver/for_fd_spec.rb +++ b/spec/ruby/library/socket/unixserver/for_fd_spec.rb @@ -1,7 +1,7 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -platform_is_not :windows do +with_feature :unix_socket do describe "UNIXServer#for_fd" do before :each do @unix_path = SocketSpecs.socket_path diff --git a/spec/ruby/library/socket/unixserver/new_spec.rb b/spec/ruby/library/socket/unixserver/new_spec.rb index f831f40bc6b..a160e3ce5c8 100644 --- a/spec/ruby/library/socket/unixserver/new_spec.rb +++ b/spec/ruby/library/socket/unixserver/new_spec.rb @@ -1,6 +1,14 @@ require_relative '../spec_helper' require_relative 'shared/new' -describe "UNIXServer.new" do - it_behaves_like :unixserver_new, :new +with_feature :unix_socket do + describe "UNIXServer.new" do + it_behaves_like :unixserver_new, :new + + it "does not use the given block and warns to use UNIXServer::open" do + -> { + @server = UNIXServer.new(@path) { raise } + }.should complain(/warning: UNIXServer::new\(\) does not take block; use UNIXServer::open\(\) instead/) + end + end end diff --git a/spec/ruby/library/socket/unixserver/open_spec.rb b/spec/ruby/library/socket/unixserver/open_spec.rb index f2506d9f6f0..16453dd3bd8 100644 --- a/spec/ruby/library/socket/unixserver/open_spec.rb +++ b/spec/ruby/library/socket/unixserver/open_spec.rb @@ -2,10 +2,10 @@ require_relative '../fixtures/classes' require_relative 'shared/new' -describe "UNIXServer.open" do - it_behaves_like :unixserver_new, :open +with_feature :unix_socket do + describe "UNIXServer.open" do + it_behaves_like :unixserver_new, :open - platform_is_not :windows do before :each do @path = SocketSpecs.socket_path end diff --git a/spec/ruby/library/socket/unixserver/shared/new.rb b/spec/ruby/library/socket/unixserver/shared/new.rb index 35395826c9c..b537f2a8713 100644 --- a/spec/ruby/library/socket/unixserver/shared/new.rb +++ b/spec/ruby/library/socket/unixserver/shared/new.rb @@ -2,21 +2,19 @@ require_relative '../../fixtures/classes' describe :unixserver_new, shared: true do - platform_is_not :windows do - before :each do - @path = SocketSpecs.socket_path - end + before :each do + @path = SocketSpecs.socket_path + end - after :each do - @server.close if @server - @server = nil - SocketSpecs.rm_socket @path - end + after :each do + @server.close if @server + @server = nil + SocketSpecs.rm_socket @path + end - it "creates a new UNIXServer" do - @server = UNIXServer.send(@method, @path) - @server.path.should == @path - @server.addr.should == ["AF_UNIX", @path] - end + it "creates a new UNIXServer" do + @server = UNIXServer.send(@method, @path) + @server.path.should == @path + @server.addr.should == ["AF_UNIX", @path] end end diff --git a/spec/ruby/library/socket/unixsocket/addr_spec.rb b/spec/ruby/library/socket/unixsocket/addr_spec.rb index e8431bea169..d93e0613123 100644 --- a/spec/ruby/library/socket/unixsocket/addr_spec.rb +++ b/spec/ruby/library/socket/unixsocket/addr_spec.rb @@ -1,9 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#addr" do - - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#addr" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) diff --git a/spec/ruby/library/socket/unixsocket/inspect_spec.rb b/spec/ruby/library/socket/unixsocket/inspect_spec.rb index d2e3cabbd36..a542ba6db54 100644 --- a/spec/ruby/library/socket/unixsocket/inspect_spec.rb +++ b/spec/ruby/library/socket/unixsocket/inspect_spec.rb @@ -1,8 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#inspect" do - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#inspect" do it "returns sockets fd for unnamed sockets" do begin s1, s2 = UNIXSocket.socketpair diff --git a/spec/ruby/library/socket/unixsocket/local_address_spec.rb b/spec/ruby/library/socket/unixsocket/local_address_spec.rb index cbf315f9f41..734253e7f5c 100644 --- a/spec/ruby/library/socket/unixsocket/local_address_spec.rb +++ b/spec/ruby/library/socket/unixsocket/local_address_spec.rb @@ -46,9 +46,7 @@ end end end -end -with_feature :unix_socket do describe 'UNIXSocket#local_address with a UNIX socket pair' do before :each do @sock, @sock2 = Socket.pair(Socket::AF_UNIX, Socket::SOCK_STREAM) diff --git a/spec/ruby/library/socket/unixsocket/new_spec.rb b/spec/ruby/library/socket/unixsocket/new_spec.rb index 05a6b3eda2e..6d8ea6dcfe6 100644 --- a/spec/ruby/library/socket/unixsocket/new_spec.rb +++ b/spec/ruby/library/socket/unixsocket/new_spec.rb @@ -1,6 +1,14 @@ require_relative '../spec_helper' require_relative 'shared/new' -describe "UNIXSocket.new" do - it_behaves_like :unixsocket_new, :new +with_feature :unix_socket do + describe "UNIXSocket.new" do + it_behaves_like :unixsocket_new, :new + + it "does not use the given block and warns to use UNIXSocket::open" do + -> { + @client = UNIXSocket.new(@path) { raise } + }.should complain(/warning: UNIXSocket::new\(\) does not take block; use UNIXSocket::open\(\) instead/) + end + end end diff --git a/spec/ruby/library/socket/unixsocket/open_spec.rb b/spec/ruby/library/socket/unixsocket/open_spec.rb index 99ad151bb83..61def30abbb 100644 --- a/spec/ruby/library/socket/unixsocket/open_spec.rb +++ b/spec/ruby/library/socket/unixsocket/open_spec.rb @@ -2,12 +2,12 @@ require_relative '../fixtures/classes' require_relative 'shared/new' -describe "UNIXSocket.open" do - it_behaves_like :unixsocket_new, :open -end +with_feature :unix_socket do + describe "UNIXSocket.open" do + it_behaves_like :unixsocket_new, :open + end -describe "UNIXSocket.open" do - platform_is_not :windows do + describe "UNIXSocket.open" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) diff --git a/spec/ruby/library/socket/unixsocket/pair_spec.rb b/spec/ruby/library/socket/unixsocket/pair_spec.rb index 0cdc55f9989..9a66c56c108 100644 --- a/spec/ruby/library/socket/unixsocket/pair_spec.rb +++ b/spec/ruby/library/socket/unixsocket/pair_spec.rb @@ -2,9 +2,8 @@ require_relative '../fixtures/classes' require_relative '../shared/partially_closable_sockets' -describe "UNIXSocket#pair" do - platform_is_not :windows do - +with_feature :unix_socket do + describe "UNIXSocket#pair" do it_should_behave_like :partially_closable_sockets before :each do diff --git a/spec/ruby/library/socket/unixsocket/partially_closable_spec.rb b/spec/ruby/library/socket/unixsocket/partially_closable_spec.rb index 4f6ef8abd92..ef7d0f0b2aa 100644 --- a/spec/ruby/library/socket/unixsocket/partially_closable_spec.rb +++ b/spec/ruby/library/socket/unixsocket/partially_closable_spec.rb @@ -2,9 +2,8 @@ require_relative '../fixtures/classes' require_relative '../shared/partially_closable_sockets' -platform_is_not :windows do +with_feature :unix_socket do describe "UNIXSocket partial closability" do - before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -20,6 +19,5 @@ end it_should_behave_like :partially_closable_sockets - end end diff --git a/spec/ruby/library/socket/unixsocket/path_spec.rb b/spec/ruby/library/socket/unixsocket/path_spec.rb index 317ffc09751..a608378e4f9 100644 --- a/spec/ruby/library/socket/unixsocket/path_spec.rb +++ b/spec/ruby/library/socket/unixsocket/path_spec.rb @@ -1,9 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#path" do - - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#path" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -24,5 +23,4 @@ @client.path.should == "" end end - end diff --git a/spec/ruby/library/socket/unixsocket/peeraddr_spec.rb b/spec/ruby/library/socket/unixsocket/peeraddr_spec.rb index 0b6b1ccf045..72bc96b1fe9 100644 --- a/spec/ruby/library/socket/unixsocket/peeraddr_spec.rb +++ b/spec/ruby/library/socket/unixsocket/peeraddr_spec.rb @@ -1,9 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#peeraddr" do - - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#peeraddr" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -26,5 +25,4 @@ }.should raise_error(Errno::ENOTCONN) end end - end diff --git a/spec/ruby/library/socket/unixsocket/recv_io_spec.rb b/spec/ruby/library/socket/unixsocket/recv_io_spec.rb index 533f02a0fac..1dbc4538e37 100644 --- a/spec/ruby/library/socket/unixsocket/recv_io_spec.rb +++ b/spec/ruby/library/socket/unixsocket/recv_io_spec.rb @@ -1,9 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#recv_io" do - - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#recv_io" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -41,9 +40,7 @@ @io.should be_an_instance_of(File) end end -end -with_feature :unix_socket do describe 'UNIXSocket#recv_io' do before do @file = File.open('/dev/null', 'w') diff --git a/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb b/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb index c0e1cf670b6..fedf74bb2fa 100644 --- a/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb +++ b/spec/ruby/library/socket/unixsocket/recvfrom_spec.rb @@ -1,8 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#recvfrom" do - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#recvfrom" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -42,10 +42,7 @@ sock.close end end -end - -with_feature :unix_socket do describe 'UNIXSocket#recvfrom' do describe 'using a socket pair' do before do diff --git a/spec/ruby/library/socket/unixsocket/send_io_spec.rb b/spec/ruby/library/socket/unixsocket/send_io_spec.rb index a2a7d265398..80f3550c6d3 100644 --- a/spec/ruby/library/socket/unixsocket/send_io_spec.rb +++ b/spec/ruby/library/socket/unixsocket/send_io_spec.rb @@ -1,9 +1,8 @@ require_relative '../spec_helper' require_relative '../fixtures/classes' -describe "UNIXSocket#send_io" do - - platform_is_not :windows do +with_feature :unix_socket do + describe "UNIXSocket#send_io" do before :each do @path = SocketSpecs.socket_path @server = UNIXServer.open(@path) @@ -32,9 +31,7 @@ @io.read.should == File.read(@send_io_path) end end -end -with_feature :unix_socket do describe 'UNIXSocket#send_io' do before do @file = File.open('/dev/null', 'w') diff --git a/spec/ruby/library/socket/unixsocket/shared/new.rb b/spec/ruby/library/socket/unixsocket/shared/new.rb index bfb7ed3886f..f075b03c5e2 100644 --- a/spec/ruby/library/socket/unixsocket/shared/new.rb +++ b/spec/ruby/library/socket/unixsocket/shared/new.rb @@ -2,23 +2,21 @@ require_relative '../../fixtures/classes' describe :unixsocket_new, shared: true do - platform_is_not :windows do - before :each do - @path = SocketSpecs.socket_path - @server = UNIXServer.open(@path) - end + before :each do + @path = SocketSpecs.socket_path + @server = UNIXServer.open(@path) + end - after :each do - @client.close if @client - @server.close - SocketSpecs.rm_socket @path - end + after :each do + @client.close if @client + @server.close + SocketSpecs.rm_socket @path + end - it "opens a unix socket on the specified file" do - @client = UNIXSocket.send(@method, @path) + it "opens a unix socket on the specified file" do + @client = UNIXSocket.send(@method, @path) - @client.addr[0].should == "AF_UNIX" - @client.should_not.closed? - end + @client.addr[0].should == "AF_UNIX" + @client.should_not.closed? end end diff --git a/spec/ruby/library/stringio/new_spec.rb b/spec/ruby/library/stringio/new_spec.rb index 328455134eb..ec4b32aa112 100644 --- a/spec/ruby/library/stringio/new_spec.rb +++ b/spec/ruby/library/stringio/new_spec.rb @@ -2,7 +2,9 @@ require 'stringio' describe "StringIO.new" do - it "warns when called with a block" do - -> { eval("StringIO.new {}") }.should complain(/StringIO::new\(\) does not take block; use StringIO::open\(\) instead/) + it "does not use the given block and warns to use StringIO::open" do + -> { + StringIO.new { raise } + }.should complain(/warning: StringIO::new\(\) does not take block; use StringIO::open\(\) instead/) end end diff --git a/spec/ruby/optional/capi/encoding_spec.rb b/spec/ruby/optional/capi/encoding_spec.rb index aa632b963b8..36437cc7b68 100644 --- a/spec/ruby/optional/capi/encoding_spec.rb +++ b/spec/ruby/optional/capi/encoding_spec.rb @@ -657,6 +657,20 @@ end end + describe "rb_enc_raise" do + it "forces exception message encoding to the specified one" do + utf_8_incompatible_string = "\x81".b + + -> { + @s.rb_enc_raise(Encoding::UTF_8, RuntimeError, utf_8_incompatible_string) + }.should raise_error { |e| + e.message.encoding.should == Encoding::UTF_8 + e.message.valid_encoding?.should == false + e.message.bytes.should == utf_8_incompatible_string.bytes + } + end + end + describe "rb_uv_to_utf8" do it 'converts a Unicode codepoint to a UTF-8 C string' do str = ' ' * 6 @@ -674,6 +688,22 @@ end end + describe "rb_enc_left_char_head" do + it 'returns the head position of a character' do + @s.rb_enc_left_char_head("é", 1).should == 0 + @s.rb_enc_left_char_head("éééé", 7).should == 6 + + @s.rb_enc_left_char_head("a", 0).should == 0 + + # unclear if this is intended to work + @s.rb_enc_left_char_head("a", 1).should == 1 + + # Works because for single-byte encodings rb_enc_left_char_head() just returns the pointer + @s.rb_enc_left_char_head("a".force_encoding(Encoding::US_ASCII), 88).should == 88 + @s.rb_enc_left_char_head("a".b, 88).should == 88 + end + end + describe "ONIGENC_MBC_CASE_FOLD" do it "returns the correct case fold for the given string" do @s.ONIGENC_MBC_CASE_FOLD("lower").should == ["l", 1] diff --git a/spec/ruby/optional/capi/ext/encoding_spec.c b/spec/ruby/optional/capi/ext/encoding_spec.c index a454ae8395f..0ebbc9d75a8 100644 --- a/spec/ruby/optional/capi/ext/encoding_spec.c +++ b/spec/ruby/optional/capi/ext/encoding_spec.c @@ -271,6 +271,13 @@ static VALUE encoding_spec_rb_enc_str_asciionly_p(VALUE self, VALUE str) { } } +static VALUE encoding_spec_rb_enc_raise(VALUE self, VALUE encoding, VALUE exception_class, VALUE format) { + rb_encoding *e = rb_to_encoding(encoding); + const char *f = RSTRING_PTR(format); + + rb_enc_raise(e, exception_class, f); +} + static VALUE encoding_spec_rb_uv_to_utf8(VALUE self, VALUE buf, VALUE num) { int len = rb_uv_to_utf8(RSTRING_PTR(buf), NUM2INT(num)); RB_ENC_CODERANGE_CLEAR(buf); @@ -307,6 +314,12 @@ static VALUE encoding_spec_rb_enc_strlen(VALUE self, VALUE str, VALUE length, VA return LONG2FIX(rb_enc_strlen(p, e, rb_to_encoding(encoding))); } +static VALUE encoding_spec_rb_enc_left_char_head(VALUE self, VALUE str, VALUE offset) { + char *ptr = RSTRING_PTR(str); + char *result = rb_enc_left_char_head(ptr, ptr + NUM2INT(offset), RSTRING_END(str), rb_enc_get(str)); + return LONG2NUM(result - ptr); +} + void Init_encoding_spec(void) { VALUE cls; native_rb_encoding_pointer = (rb_encoding**) malloc(sizeof(rb_encoding*)); @@ -362,8 +375,10 @@ void Init_encoding_spec(void) { rb_define_method(cls, "rb_enc_nth", encoding_spec_rb_enc_nth, 2); rb_define_method(cls, "rb_enc_codepoint_len", encoding_spec_rb_enc_codepoint_len, 1); rb_define_method(cls, "rb_enc_str_asciionly_p", encoding_spec_rb_enc_str_asciionly_p, 1); + rb_define_method(cls, "rb_enc_raise", encoding_spec_rb_enc_raise, 3); rb_define_method(cls, "rb_uv_to_utf8", encoding_spec_rb_uv_to_utf8, 2); rb_define_method(cls, "ONIGENC_MBC_CASE_FOLD", encoding_spec_ONIGENC_MBC_CASE_FOLD, 1); + rb_define_method(cls, "rb_enc_left_char_head", encoding_spec_rb_enc_left_char_head, 2); } #ifdef __cplusplus diff --git a/spec/ruby/optional/capi/util_spec.rb b/spec/ruby/optional/capi/util_spec.rb index 320527521bb..2c16999cdc3 100644 --- a/spec/ruby/optional/capi/util_spec.rb +++ b/spec/ruby/optional/capi/util_spec.rb @@ -127,6 +127,19 @@ ScratchPad.recorded.should == [1, 7, 4] end + it "assigns optional arguments with no hash argument given" do + @o.rb_scan_args([1, 7], "02:", 3, @acc).should == 2 + ScratchPad.recorded.should == [1, 7, nil] + end + + it "assigns optional arguments with no hash argument given and rejects the use of optional nil argument as a hash" do + -> { + @o.rb_scan_args([1, nil], "02:", 3, @acc).should == 2 + }.should_not complain + + ScratchPad.recorded.should == [1, nil, nil] + end + it "assigns required, optional, splat, post-splat, Hash and block arguments" do h = {a: 1, b: 2} @o.rb_scan_args([1, 2, 3, 4, 5, h], "k11*1:&", 6, @acc, &@prc).should == 5 diff --git a/spec/ruby/shared/string/start_with.rb b/spec/ruby/shared/string/start_with.rb index 91fc50c4cdd..4b947a3bbf0 100644 --- a/spec/ruby/shared/string/start_with.rb +++ b/spec/ruby/shared/string/start_with.rb @@ -70,7 +70,13 @@ $1.should be_nil end - ruby_bug "#19784", ""..."3.3" do + ruby_version_is ""..."3.3" do + it "does not check that we are not matching part of a character" do + "\xC3\xA9".send(@method).should.start_with?("\xC3") + end + end + + ruby_version_is "3.3" do # #19784 it "checks that we are not matching part of a character" do "\xC3\xA9".send(@method).should_not.start_with?("\xC3") end From cf4e404ecf61ac1eb88b1edc74e946f84b8f9a19 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 30 Oct 2023 14:10:35 +0100 Subject: [PATCH 46/56] Add tags for new failing specs --- spec/tags/ruby/core/io/gets_tags.txt | 1 + spec/tags/ruby/core/io/pread_tags.txt | 2 ++ spec/tags/ruby/core/io/pwrite_tags.txt | 2 ++ spec/tags/ruby/core/kernel/require_tags.txt | 2 ++ .../ruby/core/module/define_method_tags.txt | 1 + spec/tags/ruby/core/signal/trap_tags.txt | 1 + .../library/openssl/digest/append_tags.txt | 8 +++++ .../openssl/digest/initialize_tags.txt | 13 +++++++++ .../ruby/library/openssl/digest/name_tags.txt | 1 + .../library/openssl/digest/update_tags.txt | 8 +++++ .../library/openssl/kdf/pbkdf2_hmac_tags.txt | 9 ++++++ .../ruby/library/openssl/kdf/scrypt_tags.txt | 29 +++++++++++++++++++ 12 files changed, 77 insertions(+) create mode 100644 spec/tags/ruby/core/io/pread_tags.txt create mode 100644 spec/tags/ruby/core/io/pwrite_tags.txt create mode 100644 spec/tags/ruby/library/openssl/digest/append_tags.txt create mode 100644 spec/tags/ruby/library/openssl/digest/initialize_tags.txt create mode 100644 spec/tags/ruby/library/openssl/digest/name_tags.txt create mode 100644 spec/tags/ruby/library/openssl/digest/update_tags.txt create mode 100644 spec/tags/ruby/library/openssl/kdf/pbkdf2_hmac_tags.txt create mode 100644 spec/tags/ruby/library/openssl/kdf/scrypt_tags.txt diff --git a/spec/tags/ruby/core/io/gets_tags.txt b/spec/tags/ruby/core/io/gets_tags.txt index 37c681ee974..42011ccde84 100644 --- a/spec/tags/ruby/core/io/gets_tags.txt +++ b/spec/tags/ruby/core/io/gets_tags.txt @@ -1,3 +1,4 @@ windows:IO#gets ignores the internal encoding if the default external encoding is ASCII-8BIT windows:IO#gets transcodes to internal encoding if the IO object's external encoding is ASCII-8BIT fails:IO#gets when passed chomp raises exception when options passed as Hash +fails:IO#gets sets $_ to nil after the last line has been read diff --git a/spec/tags/ruby/core/io/pread_tags.txt b/spec/tags/ruby/core/io/pread_tags.txt new file mode 100644 index 00000000000..3110b947dba --- /dev/null +++ b/spec/tags/ruby/core/io/pread_tags.txt @@ -0,0 +1,2 @@ +fails:IO#pread converts maxlen to Integer using #to_int +fails:IO#pread raises ArgumentError for negative values of maxlen diff --git a/spec/tags/ruby/core/io/pwrite_tags.txt b/spec/tags/ruby/core/io/pwrite_tags.txt new file mode 100644 index 00000000000..0c9a41052f5 --- /dev/null +++ b/spec/tags/ruby/core/io/pwrite_tags.txt @@ -0,0 +1,2 @@ +fails:IO#pwrite calls #to_s on the object to be written +fails:IO#pwrite raises a NoMethodError if object does not respond to #to_s diff --git a/spec/tags/ruby/core/kernel/require_tags.txt b/spec/tags/ruby/core/kernel/require_tags.txt index eb3c91aa2d1..5dbb139e7f7 100644 --- a/spec/tags/ruby/core/kernel/require_tags.txt +++ b/spec/tags/ruby/core/kernel/require_tags.txt @@ -7,3 +7,5 @@ fails:Kernel.require ($LOADED_FEATURES) with symlinks in the required feature an fails:Kernel#require ($LOADED_FEATURES) unicode_normalize is part of core and not $LOADED_FEATURES fails:Kernel.require ($LOADED_FEATURES) unicode_normalize is part of core and not $LOADED_FEATURES fails:Kernel#require complex, enumerator, rational, thread, ruby2_keywords, fiber are already required +fails:Kernel#require (path resolution) loads c-extension file when passed absolute path without extension when no .rb is present +fails:Kernel.require (path resolution) loads c-extension file when passed absolute path without extension when no .rb is present diff --git a/spec/tags/ruby/core/module/define_method_tags.txt b/spec/tags/ruby/core/module/define_method_tags.txt index 87105537d21..11e6aedea4d 100644 --- a/spec/tags/ruby/core/module/define_method_tags.txt +++ b/spec/tags/ruby/core/module/define_method_tags.txt @@ -4,3 +4,4 @@ fails:Method#define_method when passed a block behaves exactly like a lambda for fails:Method#define_method when passed a block behaves exactly like a lambda for next fails:Module#define_method raises TypeError when #to_str called on non-String name returns non-String value fails:Module#define_method raises a TypeError when the given method is no Method/Proc +fails:Module#define_method when passed a Proc object and a method is defined inside defines the nested method in the default definee where the Proc was created diff --git a/spec/tags/ruby/core/signal/trap_tags.txt b/spec/tags/ruby/core/signal/trap_tags.txt index 89cb4130017..09d157adbd8 100644 --- a/spec/tags/ruby/core/signal/trap_tags.txt +++ b/spec/tags/ruby/core/signal/trap_tags.txt @@ -8,3 +8,4 @@ fails:Signal.trap accepts 'SYSTEM_DEFAULT' and uses the OS handler for SIGPIPE fails:Signal.trap allows to register a handler for all known signals, except reserved signals for which it raises ArgumentError fails:Signal.trap raises ArgumentError when passed unknown signal +fails:Signal.trap calls #to_str on an object to convert to a String diff --git a/spec/tags/ruby/library/openssl/digest/append_tags.txt b/spec/tags/ruby/library/openssl/digest/append_tags.txt new file mode 100644 index 00000000000..4ba14aae284 --- /dev/null +++ b/spec/tags/ruby/library/openssl/digest/append_tags.txt @@ -0,0 +1,8 @@ +fails:OpenSSL::Digest#<< when input is not a String and responds to #to_str returns a SHA1 digest +fails:OpenSSL::Digest#<< when input is not a String and responds to #to_str returns a SHA256 digest +fails:OpenSSL::Digest#<< when input is not a String and responds to #to_str returns a SHA384 digest +fails:OpenSSL::Digest#<< when input is not a String and responds to #to_str returns a SHA512 digest +fails:OpenSSL::Digest#<< when input is not a String and does not respond to #to_str raises a TypeError with SHA1 +fails:OpenSSL::Digest#<< when input is not a String and does not respond to #to_str raises a TypeError with SHA256 +fails:OpenSSL::Digest#<< when input is not a String and does not respond to #to_str raises a TypeError with SHA384 +fails:OpenSSL::Digest#<< when input is not a String and does not respond to #to_str raises a TypeError with SHA512 diff --git a/spec/tags/ruby/library/openssl/digest/initialize_tags.txt b/spec/tags/ruby/library/openssl/digest/initialize_tags.txt new file mode 100644 index 00000000000..6d25bb07e05 --- /dev/null +++ b/spec/tags/ruby/library/openssl/digest/initialize_tags.txt @@ -0,0 +1,13 @@ +fails:OpenSSL::Digest#initialize cannot be called with a digest class +fails:OpenSSL::Digest#initialize can be called with a digest name returns a SHA1 object +fails:OpenSSL::Digest#initialize can be called with a digest name returns a SHA256 object +fails:OpenSSL::Digest#initialize can be called with a digest name returns a SHA384 object +fails:OpenSSL::Digest#initialize can be called with a digest name returns a SHA512 object +fails:OpenSSL::Digest#initialize can be called with a digest name throws an error when called with an unknown digest +fails:OpenSSL::Digest#initialize can be called with a digest name cannot be called with a symbol +fails:OpenSSL::Digest#initialize can be called with a digest name does not call #to_str on the argument +fails:OpenSSL::Digest#initialize can be called with a digest object returns a SHA1 object +fails:OpenSSL::Digest#initialize can be called with a digest object returns a SHA256 object +fails:OpenSSL::Digest#initialize can be called with a digest object returns a SHA384 object +fails:OpenSSL::Digest#initialize can be called with a digest object returns a SHA512 object +fails:OpenSSL::Digest#initialize can be called with a digest object ignores the state of the digest object diff --git a/spec/tags/ruby/library/openssl/digest/name_tags.txt b/spec/tags/ruby/library/openssl/digest/name_tags.txt new file mode 100644 index 00000000000..c81a7dcadd6 --- /dev/null +++ b/spec/tags/ruby/library/openssl/digest/name_tags.txt @@ -0,0 +1 @@ +fails:OpenSSL::Digest#name converts the name to the internal representation of OpenSSL diff --git a/spec/tags/ruby/library/openssl/digest/update_tags.txt b/spec/tags/ruby/library/openssl/digest/update_tags.txt new file mode 100644 index 00000000000..9159dac950e --- /dev/null +++ b/spec/tags/ruby/library/openssl/digest/update_tags.txt @@ -0,0 +1,8 @@ +fails:OpenSSL::Digest#update when input is not a String and responds to #to_str returns a SHA1 digest +fails:OpenSSL::Digest#update when input is not a String and responds to #to_str returns a SHA256 digest +fails:OpenSSL::Digest#update when input is not a String and responds to #to_str returns a SHA384 digest +fails:OpenSSL::Digest#update when input is not a String and responds to #to_str returns a SHA512 digest +fails:OpenSSL::Digest#update when input is not a String and does not respond to #to_str raises a TypeError with SHA1 +fails:OpenSSL::Digest#update when input is not a String and does not respond to #to_str raises a TypeError with SHA256 +fails:OpenSSL::Digest#update when input is not a String and does not respond to #to_str raises a TypeError with SHA384 +fails:OpenSSL::Digest#update when input is not a String and does not respond to #to_str raises a TypeError with SHA512 diff --git a/spec/tags/ruby/library/openssl/kdf/pbkdf2_hmac_tags.txt b/spec/tags/ruby/library/openssl/kdf/pbkdf2_hmac_tags.txt new file mode 100644 index 00000000000..ed329d16c9d --- /dev/null +++ b/spec/tags/ruby/library/openssl/kdf/pbkdf2_hmac_tags.txt @@ -0,0 +1,9 @@ +fails:OpenSSL::KDF.pbkdf2_hmac treats all keywords as required +fails:OpenSSL::KDF.pbkdf2_hmac raises an OpenSSL::KDF::KDFError for 0 or less iterations +fails:OpenSSL::KDF.pbkdf2_hmac raises a TypeError when hash is neither a String nor an OpenSSL::Digest +fails:OpenSSL::KDF.pbkdf2_hmac raises a TypeError when hash is neither a String nor an OpenSSL::Digest, it does not try to call #to_str +fails:OpenSSL::KDF.pbkdf2_hmac raises a RuntimeError for unknown digest algorithms +fails:OpenSSL::KDF.pbkdf2_hmac treats salt as a required keyword +fails:OpenSSL::KDF.pbkdf2_hmac treats iterations as a required keyword +fails:OpenSSL::KDF.pbkdf2_hmac treats length as a required keyword +fails:OpenSSL::KDF.pbkdf2_hmac treats hash as a required keyword diff --git a/spec/tags/ruby/library/openssl/kdf/scrypt_tags.txt b/spec/tags/ruby/library/openssl/kdf/scrypt_tags.txt new file mode 100644 index 00000000000..08c626ff287 --- /dev/null +++ b/spec/tags/ruby/library/openssl/kdf/scrypt_tags.txt @@ -0,0 +1,29 @@ +fails:OpenSSL::KDF.scrypt creates the same value with the same input +fails:OpenSSL::KDF.scrypt supports nullbytes embedded into the password +fails:OpenSSL::KDF.scrypt coerces the password into a String using #to_str +fails:OpenSSL::KDF.scrypt coerces the salt into a String using #to_str +fails:OpenSSL::KDF.scrypt coerces the N into an Integer using #to_int +fails:OpenSSL::KDF.scrypt coerces the r into an Integer using #to_int +fails:OpenSSL::KDF.scrypt coerces the p into an Integer using #to_int +fails:OpenSSL::KDF.scrypt coerces the length into an Integer using #to_int +fails:OpenSSL::KDF.scrypt accepts an empty password +fails:OpenSSL::KDF.scrypt accepts an empty salt +fails:OpenSSL::KDF.scrypt accepts a zero length +fails:OpenSSL::KDF.scrypt accepts an arbitrary length +fails:OpenSSL::KDF.scrypt raises a TypeError when password is not a String and does not respond to #to_str +fails:OpenSSL::KDF.scrypt raises a TypeError when salt is not a String and does not respond to #to_str +fails:OpenSSL::KDF.scrypt raises a TypeError when N is not an Integer and does not respond to #to_int +fails:OpenSSL::KDF.scrypt raises a TypeError when r is not an Integer and does not respond to #to_int +fails:OpenSSL::KDF.scrypt raises a TypeError when p is not an Integer and does not respond to #to_int +fails:OpenSSL::KDF.scrypt raises a TypeError when length is not an Integer and does not respond to #to_int +fails:OpenSSL::KDF.scrypt treats salt as a required keyword +fails:OpenSSL::KDF.scrypt treats N as a required keyword +fails:OpenSSL::KDF.scrypt treats r as a required keyword +fails:OpenSSL::KDF.scrypt treats p as a required keyword +fails:OpenSSL::KDF.scrypt treats length as a required keyword +fails:OpenSSL::KDF.scrypt treats all keywords as required +fails:OpenSSL::KDF.scrypt requires N to be a power of 2 +fails:OpenSSL::KDF.scrypt requires N to be at least 2 +fails:OpenSSL::KDF.scrypt requires r to be positive +fails:OpenSSL::KDF.scrypt requires p to be positive +fails:OpenSSL::KDF.scrypt requires length to be not negative From ee280f8b353331db8a51f4cac0432a2657f5431d Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 30 Oct 2023 14:17:21 +0100 Subject: [PATCH 47/56] Fix logic for finding tag file from spec description --- spec/mspec/tool/tag_from_output.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/mspec/tool/tag_from_output.rb b/spec/mspec/tool/tag_from_output.rb index a6e60945cdf..d8d9efd2816 100755 --- a/spec/mspec/tool/tag_from_output.rb +++ b/spec/mspec/tool/tag_from_output.rb @@ -30,9 +30,9 @@ if spec_file spec_file = spec_file[SPEC_FILE, 1] or raise else - if error_line =~ /^(\w+)[#\.](\w+) / - module_method = error_line.split(' ', 2).first - file = "#{$1.downcase}/#{$2}_spec.rb" + if error_line =~ /^([\w:]+)[#\.](\w+) / + mod, method = $1, $2 + file = "#{mod.downcase.gsub('::', '/')}/#{method}_spec.rb" spec_file = ['spec/ruby/core', 'spec/ruby/library', *Dir.glob('spec/ruby/library/*')].find { |dir| path = "#{dir}/#{file}" break path if File.exist?(path) From 7f80aeb74d9a90be9eb60f10c0a8edb6cd5b92ab Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 30 Oct 2023 14:39:39 +0100 Subject: [PATCH 48/56] Add tags for specs failing with JIT --- spec/tags/ruby/core/module/define_method_tags.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/tags/ruby/core/module/define_method_tags.txt b/spec/tags/ruby/core/module/define_method_tags.txt index 11e6aedea4d..8e4ff27a6d4 100644 --- a/spec/tags/ruby/core/module/define_method_tags.txt +++ b/spec/tags/ruby/core/module/define_method_tags.txt @@ -5,3 +5,5 @@ fails:Method#define_method when passed a block behaves exactly like a lambda for fails:Module#define_method raises TypeError when #to_str called on non-String name returns non-String value fails:Module#define_method raises a TypeError when the given method is no Method/Proc fails:Module#define_method when passed a Proc object and a method is defined inside defines the nested method in the default definee where the Proc was created +fails:Module#define_method when passed a block behaves exactly like a lambda for break +fails:Module#define_method when passed a block behaves exactly like a lambda for next From 830c0dc0f0c916bbbcad0ad001d0a288f93350d0 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 31 Oct 2023 12:53:50 -0500 Subject: [PATCH 49/56] Preserve insertion order for ivars I could not find anywhere this is specified, except for: * A very recent test in CRuby's test/ruby/test_shape.rb that confirms a degraded "complex" shape using CRuby's "st table" hashtable will continue to be returned in insertion order. This implies that insertion order is expected. * A rubyspec added in 2021 in instance_variable_spec.rb that confirms instance variables are returned in insertion order. No links to issues or discussions is provided. Since this is a simple enough change, we'll go ahead with it, but I don't know that it has ever been formally specified (and indeed I looked through other tests of other types of variables and see that there are commits to .sort them before inspection in tests.) Fixes #7988 --- .../java/org/jruby/runtime/ivars/VariableTableManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/jruby/runtime/ivars/VariableTableManager.java b/core/src/main/java/org/jruby/runtime/ivars/VariableTableManager.java index d16fb7b50fe..58fc8a89631 100644 --- a/core/src/main/java/org/jruby/runtime/ivars/VariableTableManager.java +++ b/core/src/main/java/org/jruby/runtime/ivars/VariableTableManager.java @@ -33,7 +33,7 @@ import java.lang.invoke.MethodHandle; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.function.Function; @@ -258,7 +258,7 @@ VariableAccessor getVariableAccessorWithBuilder(String name, Function newVariableAccessors = new HashMap(myVariableAccessors.size() + 1); + Map newVariableAccessors = new LinkedHashMap<>(myVariableAccessors.size() + 1); newVariableAccessors.putAll(myVariableAccessors); newVariableAccessors.put(name, ivarAccessor); @@ -390,7 +390,7 @@ public int getVariableTableSizeWithExtras() { * @return a map of names to accessors for all variables */ public Map getVariableTableCopy() { - return new HashMap(getVariableAccessorsForRead()); + return new LinkedHashMap(getVariableAccessorsForRead()); } /** From 34ef8d54e6b1c686259ee94fd2613054b44d7846 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 31 Oct 2023 12:58:02 -0500 Subject: [PATCH 50/56] Untag passing instance_variables ordering spec --- spec/tags/ruby/core/kernel/instance_variables_tags.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 spec/tags/ruby/core/kernel/instance_variables_tags.txt diff --git a/spec/tags/ruby/core/kernel/instance_variables_tags.txt b/spec/tags/ruby/core/kernel/instance_variables_tags.txt deleted file mode 100644 index df860386d01..00000000000 --- a/spec/tags/ruby/core/kernel/instance_variables_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:Kernel#instance_variables regular objects returns the instances variables in the order declared From fef81e6d0a4f59915f346c08746e95cf6574fec1 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Tue, 31 Oct 2023 14:38:51 -0500 Subject: [PATCH 51/56] Guard Fiber::Scheduler support behind a property This commit adds the property "jruby.experimental.fiber.scheduler" to control whether the Fiber::Scheduler subsystem will be used at runtime. It is currently off by default, as this is an experimental feature in all versions of Ruby that support it. Set this property at the JVM level, or pass -Xexperimental.fiber.scheduler to enable it. None of the classes or constants associated with the fiber scheduler will be defined if the property is not enabled. In the future, this property may disappear when this feature is finalized, or may remain under a different name to allow disabling all scheduler checks. --- core/src/main/java/org/jruby/Ruby.java | 19 ++++-- core/src/main/java/org/jruby/RubyIO.java | 11 ++-- .../src/main/java/org/jruby/RubyIOBuffer.java | 1 + .../java/org/jruby/ext/fiber/ThreadFiber.java | 58 +++++++++--------- .../jruby/ext/fiber/ThreadFiberLibrary.java | 6 ++ .../java/org/jruby/util/cli/Category.java | 3 +- .../main/java/org/jruby/util/cli/Options.java | 2 + .../main/java/org/jruby/util/io/OpenFile.java | 61 +++++++++++-------- rakelib/test.rake | 6 +- 9 files changed, 100 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/jruby/Ruby.java b/core/src/main/java/org/jruby/Ruby.java index f24ecba7807..4736f50fb18 100644 --- a/core/src/main/java/org/jruby/Ruby.java +++ b/core/src/main/java/org/jruby/Ruby.java @@ -443,7 +443,11 @@ private Ruby(RubyInstanceConfig config) { randomClass = null; } ioClass = RubyIO.createIOClass(this); - ioBufferClass = RubyIOBuffer.createIOBufferClass(this); + if (Options.FIBER_SCHEDULER.load()) { + ioBufferClass = RubyIOBuffer.createIOBufferClass(this); + } else { + ioBufferClass = null; + } structClass = profile.allowClass("Struct") ? RubyStruct.createStructClass(this) : null; bindingClass = profile.allowClass("Binding") ? RubyBinding.createBindingClass(this) : null; @@ -1718,11 +1722,14 @@ private void initExceptions() { RubyClass runtimeError = this.runtimeError; ObjectAllocator runtimeErrorAllocator = runtimeError.getAllocator(); - bufferLockedError = ioBufferClass.defineClassUnder("LockedError", runtimeError, runtimeErrorAllocator); - bufferAllocationError = ioBufferClass.defineClassUnder("AllocationError", runtimeError, runtimeErrorAllocator); - bufferAccessError = ioBufferClass.defineClassUnder("AccessError", runtimeError, runtimeErrorAllocator); - bufferInvalidatedError = ioBufferClass.defineClassUnder("InvalidatedError", runtimeError, runtimeErrorAllocator); - bufferMaskError = ioBufferClass.defineClassUnder("MaskError", runtimeError, runtimeErrorAllocator); + + if (Options.FIBER_SCHEDULER.load()) { + bufferLockedError = ioBufferClass.defineClassUnder("LockedError", runtimeError, runtimeErrorAllocator); + bufferAllocationError = ioBufferClass.defineClassUnder("AllocationError", runtimeError, runtimeErrorAllocator); + bufferAccessError = ioBufferClass.defineClassUnder("AccessError", runtimeError, runtimeErrorAllocator); + bufferInvalidatedError = ioBufferClass.defineClassUnder("InvalidatedError", runtimeError, runtimeErrorAllocator); + bufferMaskError = ioBufferClass.defineClassUnder("MaskError", runtimeError, runtimeErrorAllocator); + } initErrno(); diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java index 1fd0e2c8155..e224f8c9b91 100644 --- a/core/src/main/java/org/jruby/RubyIO.java +++ b/core/src/main/java/org/jruby/RubyIO.java @@ -86,6 +86,7 @@ import org.jruby.runtime.encoding.EncodingService; import org.jruby.util.ShellLauncher.POpenProcess; import org.jruby.util.*; +import org.jruby.util.cli.Options; import org.jruby.util.io.ChannelFD; import org.jruby.util.io.EncodingUtils; import org.jruby.util.io.FilenoUtil; @@ -3795,10 +3796,12 @@ public static RubyIO convertToIO(ThreadContext context, IRubyObject obj) { @JRubyMethod(name = "select", required = 1, optional = 3, checkArity = false, meta = true) public static IRubyObject select(ThreadContext context, IRubyObject recv, IRubyObject[] argv) { - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioSelectv(context, scheduler, argv); - if (result != UNDEF) return result; + if (Options.FIBER_SCHEDULER.load()) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioSelectv(context, scheduler, argv); + if (result != UNDEF) return result; + } } int argc = Arity.checkArgumentCount(context, argv, 1, 4); diff --git a/core/src/main/java/org/jruby/RubyIOBuffer.java b/core/src/main/java/org/jruby/RubyIOBuffer.java index 5d5e13e6667..dfe2916cf58 100644 --- a/core/src/main/java/org/jruby/RubyIOBuffer.java +++ b/core/src/main/java/org/jruby/RubyIOBuffer.java @@ -14,6 +14,7 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.ByteList; +import org.jruby.util.cli.Options; import org.jruby.util.io.ChannelFD; import org.jruby.util.io.OpenFile; diff --git a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java index c676c7dd64b..7844b49fa5d 100644 --- a/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java +++ b/core/src/main/java/org/jruby/ext/fiber/ThreadFiber.java @@ -691,38 +691,40 @@ public IRubyObject backtrace_locations(ThreadContext context, IRubyObject level, return threadFiber.thread.backtrace_locations(context, level, length); } - // MRI: rb_fiber_s_schedule_kw and rb_fiber_s_schedule, kw passes on context - @JRubyMethod(name = "schedule", meta = true, rest = true, keywords = true) - public static IRubyObject schedule(ThreadContext context, IRubyObject self, IRubyObject[] args, Block block) { - RubyThread thread = context.getThread(); - IRubyObject scheduler = thread.getScheduler(); - IRubyObject fiber = context.nil; - - if (!scheduler.isNil()) { - fiber = scheduler.callMethod(context, "fiber", args, block); - } else { - throw context.runtime.newRuntimeError("No scheduler is available!"); - } + public static class FiberSchedulerSupport { + // MRI: rb_fiber_s_schedule_kw and rb_fiber_s_schedule, kw passes on context + @JRubyMethod(name = "schedule", meta = true, rest = true, keywords = true) + public static IRubyObject schedule(ThreadContext context, IRubyObject self, IRubyObject[] args, Block block) { + RubyThread thread = context.getThread(); + IRubyObject scheduler = thread.getScheduler(); + IRubyObject fiber = context.nil; + + if (!scheduler.isNil()) { + fiber = scheduler.callMethod(context, "fiber", args, block); + } else { + throw context.runtime.newRuntimeError("No scheduler is available!"); + } - return fiber; - } + return fiber; + } - // MRI: rb_fiber_s_scheduler - @JRubyMethod(name = "scheduler", meta = true) - public static IRubyObject get_scheduler(ThreadContext context, IRubyObject self) { - return context.getFiberCurrentThread().getScheduler(); - } + // MRI: rb_fiber_s_scheduler + @JRubyMethod(name = "scheduler", meta = true) + public static IRubyObject get_scheduler(ThreadContext context, IRubyObject self) { + return context.getFiberCurrentThread().getScheduler(); + } - // MRI: rb_fiber_current_scheduler - @JRubyMethod(name = "current_scheduler", meta = true) - public static IRubyObject current_scheduler(ThreadContext context, IRubyObject self) { - return context.getFiberCurrentThread().getSchedulerCurrent(); - } + // MRI: rb_fiber_current_scheduler + @JRubyMethod(name = "current_scheduler", meta = true) + public static IRubyObject current_scheduler(ThreadContext context, IRubyObject self) { + return context.getFiberCurrentThread().getSchedulerCurrent(); + } - // MRI: rb_fiber_set_scheduler - @JRubyMethod(name = "set_scheduler", meta = true) - public static IRubyObject set_scheduler(ThreadContext context, IRubyObject self, IRubyObject scheduler) { - return context.getFiberCurrentThread().setFiberScheduler(scheduler); + // MRI: rb_fiber_set_scheduler + @JRubyMethod(name = "set_scheduler", meta = true) + public static IRubyObject set_scheduler(ThreadContext context, IRubyObject self, IRubyObject scheduler) { + return context.getFiberCurrentThread().setFiberScheduler(scheduler); + } } public FiberData getData() { diff --git a/core/src/main/java/org/jruby/ext/fiber/ThreadFiberLibrary.java b/core/src/main/java/org/jruby/ext/fiber/ThreadFiberLibrary.java index e6582249d6a..92d2c722269 100644 --- a/core/src/main/java/org/jruby/ext/fiber/ThreadFiberLibrary.java +++ b/core/src/main/java/org/jruby/ext/fiber/ThreadFiberLibrary.java @@ -33,6 +33,7 @@ import org.jruby.runtime.ObjectAllocator; import org.jruby.runtime.load.Library; import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.cli.Options; /** * A thread-based implementation of Ruby 1.9 Fiber library. @@ -43,6 +44,11 @@ public RubyClass createFiberClass(final Ruby runtime) { cFiber.defineAnnotatedMethods(ThreadFiber.class); + if (Options.FIBER_SCHEDULER.load()) { + // define additional methods for Fiber::Scheduler support + cFiber.defineAnnotatedMethods(ThreadFiber.FiberSchedulerSupport.class); + } + return cFiber; } } diff --git a/core/src/main/java/org/jruby/util/cli/Category.java b/core/src/main/java/org/jruby/util/cli/Category.java index 934ef1e5432..9df89cb3dd3 100644 --- a/core/src/main/java/org/jruby/util/cli/Category.java +++ b/core/src/main/java/org/jruby/util/cli/Category.java @@ -46,7 +46,8 @@ public enum Category { JAVA_INTEGRATION("java integration"), PROFILING("profiling"), CLI("command line options"), - COMPLIANCE("compliance options"); + COMPLIANCE("compliance options"), + EXPERIMENTAL("experimental features"); Category(String desc) { this.desc = desc; diff --git a/core/src/main/java/org/jruby/util/cli/Options.java b/core/src/main/java/org/jruby/util/cli/Options.java index 77492f731d8..c2a542a3885 100644 --- a/core/src/main/java/org/jruby/util/cli/Options.java +++ b/core/src/main/java/org/jruby/util/cli/Options.java @@ -193,6 +193,8 @@ public class Options { public static final Option PROFILE_MAX_METHODS = integer(PROFILING, "profile.max.methods", 100000, "Maximum number of methods to consider for profiling."); + public static final Option FIBER_SCHEDULER = bool(EXPERIMENTAL, "experimental.fiber.scheduler", false, "Enable experimental Fiber::Scheduler support."); + public static final Option CLI_AUTOSPLIT = bool(CLI, "cli.autosplit", false, "Split $_ into $F for -p or -n. Same as -a."); public static final Option CLI_DEBUG = bool(CLI, "cli.debug", false, "Enable debug mode logging. Same as -d."); public static final Option CLI_PROCESS_LINE_ENDS = bool(CLI, "cli.process.line.ends", false, "Enable line ending processing. Same as -l."); diff --git a/core/src/main/java/org/jruby/util/io/OpenFile.java b/core/src/main/java/org/jruby/util/io/OpenFile.java index 6a3ec3a7cd7..87ea1e962c8 100644 --- a/core/src/main/java/org/jruby/util/io/OpenFile.java +++ b/core/src/main/java/org/jruby/util/io/OpenFile.java @@ -28,7 +28,6 @@ import org.jruby.Finalizable; import org.jruby.Ruby; import org.jruby.RubyArgsFile; -import org.jruby.RubyBasicObject; import org.jruby.RubyBignum; import org.jruby.RubyEncoding; import org.jruby.RubyException; @@ -48,6 +47,7 @@ import org.jruby.util.ByteList; import org.jruby.util.ShellLauncher; import org.jruby.util.StringSupport; +import org.jruby.util.cli.Options; import static org.jruby.util.StringSupport.*; @@ -61,6 +61,7 @@ public OpenFile(RubyIO io, IRubyObject nil) { writeconvPreEcopts = nil; encs.ecopts = nil; posix = new PosixShim(runtime); + fiberScheduler = Options.FIBER_SCHEDULER.load(); } // IO Mode flags @@ -162,6 +163,8 @@ public static class Buffer { private final Ptr spPtr = new Ptr(); private final Ptr dpPtr = new Ptr(); + private final boolean fiberScheduler; + public void clearStdio() { stdio_file = null; } @@ -462,7 +465,7 @@ public int io_fflush(ThreadContext context) { // rb_io_wait_writable public boolean waitWritable(ThreadContext context, long timeout) { - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + IRubyObject scheduler = fiberScheduler ? context.getFiberCurrentThread().getSchedulerCurrent() : null; boolean locked = lock(); try { @@ -477,7 +480,7 @@ public boolean waitWritable(ThreadContext context, long timeout) { return true; case EAGAIN: case EWOULDBLOCK: - if (!scheduler.isNil()) { + if (fiberScheduler && !scheduler.isNil()) { return FiberScheduler.ioWaitWritable(context, scheduler, RubyIO.newIO(context.runtime, channel())).isTrue(); } @@ -498,7 +501,7 @@ public boolean waitWritable(ThreadContext context) { // rb_io_wait_readable public boolean waitReadable(ThreadContext context, long timeout) { - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + IRubyObject scheduler = fiberScheduler ? context.getFiberCurrentThread().getSchedulerCurrent() : null; boolean locked = lock(); try { @@ -513,7 +516,7 @@ public boolean waitReadable(ThreadContext context, long timeout) { return true; case EAGAIN: case EWOULDBLOCK: - if (!scheduler.isNil()) { + if (fiberScheduler && !scheduler.isNil()) { return FiberScheduler.ioWaitReadable(context, scheduler, RubyIO.newIO(context.runtime, channel())).isTrue(); } @@ -1478,12 +1481,14 @@ public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD f // rb_io_buffer_read_internal public static int readInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer buffer, int buf, int count) { // try scheduler first - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.io, buffer, buf, count); + if (fptr.fiberScheduler) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioReadMemory(context, scheduler, fptr.io, buffer, buf, count); - if (result != null) { - return FiberScheduler.resultApply(context, result); + if (result != null) { + return FiberScheduler.resultApply(context, result); + } } } @@ -1513,12 +1518,14 @@ simple read(2) because EINTR does not damage the descriptor. public static int preadInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer buffer, int from, int length) { // try scheduler first - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioPReadMemory(context, scheduler, fptr.io, buffer, from, length, 0); + if (fptr.fiberScheduler) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPReadMemory(context, scheduler, fptr.io, buffer, from, length, 0); - if (result != null) { - return FiberScheduler.resultApply(context, result); + if (result != null) { + return FiberScheduler.resultApply(context, result); + } } } @@ -1532,12 +1539,14 @@ public static int preadInternal(ThreadContext context, OpenFile fptr, ChannelFD public static int pwriteInternal(ThreadContext context, OpenFile fptr, ChannelFD fd, ByteBuffer buffer, int from, int length) { // try scheduler first - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioPWriteMemory(context, scheduler, fptr.io, buffer, from, length, 0); + if (fptr.fiberScheduler) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioPWriteMemory(context, scheduler, fptr.io, buffer, from, length, 0); - if (result != null) { - return FiberScheduler.resultApply(context, result); + if (result != null) { + return FiberScheduler.resultApply(context, result); + } } } @@ -2479,12 +2488,14 @@ public static int writeInternal(ThreadContext context, OpenFile fptr, byte[] buf // rb_io_buffer_write_internal public static int writeInternal(ThreadContext context, OpenFile fptr, ByteBuffer bufBytes, int buf, int count) { - IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); - if (!scheduler.isNil()) { - IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.io, bufBytes, buf, count); + if (fptr.fiberScheduler) { + IRubyObject scheduler = context.getFiberCurrentThread().getSchedulerCurrent(); + if (!scheduler.isNil()) { + IRubyObject result = FiberScheduler.ioWriteMemory(context, scheduler, fptr.io, bufBytes, buf, count); - if (result != null) { - return FiberScheduler.resultApply(context, result); + if (result != null) { + return FiberScheduler.resultApply(context, result); + } } } diff --git a/rakelib/test.rake b/rakelib/test.rake index 1f531dee797..51b20dc86d6 100644 --- a/rakelib/test.rake +++ b/rakelib/test.rake @@ -100,9 +100,9 @@ namespace :test do mri_suites = [:core, :extra, :stdlib] mri_suites = { - core: "-Xbacktrace.style=mri -Xdebug.fullTrace", - extra: "--disable-gems -Xbacktrace.style=mri -Xdebug.fullTrace", - stdlib: "-Xbacktrace.style=mri -Xdebug.fullTrace", + core: "-Xbacktrace.style=mri -Xdebug.fullTrace -Xexperimental.fiber.scheduler", + extra: "--disable-gems -Xbacktrace.style=mri -Xdebug.fullTrace -Xexperimental.fiber.scheduler", + stdlib: "-Xbacktrace.style=mri -Xdebug.fullTrace -Xexperimental.fiber.scheduler", } mri_suites.each do |suite, extra_jruby_opts| From 5b2057461b55bd7b7de0b3ff4ae47b81d3e76adc Mon Sep 17 00:00:00 2001 From: "Thomas E. Enebo" Date: Wed, 1 Nov 2023 15:46:27 -0400 Subject: [PATCH 52/56] Tweak release script --- tool/release.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tool/release.sh b/tool/release.sh index 34ae355e35e..b9b80f08675 100644 --- a/tool/release.sh +++ b/tool/release.sh @@ -5,15 +5,16 @@ # 2. Assumes jruby you are using in your PATH works set -x -export JRUBY_VERSION=9.4.4.0 -export WINDOWS_VERSION=9_4_4_0 +export REPO=${REPO:=jruby} +export JRUBY_VERSION=${JRUBY_VERSION:=9.4.5.0} +export WINDOWS_VERSION=`echo $JRUBY_VERSION|sed -e 's/\./_/g'` echo $JRUBY_VERSION > VERSION mvn git add VERSION core/pom.xml lib/pom.xml pom.xml shaded/pom.xml git commit -m "Version $JRUBY_VERSION updated for release" cd .. rm -rf release -git clone jruby release +git clone $REPO release cd release pwd mvn clean deploy -Psonatype-oss-release -Prelease From 1abae2700ffd6ddec93b661400c9744e9bb45eff Mon Sep 17 00:00:00 2001 From: "Thomas E. Enebo" Date: Thu, 2 Nov 2023 10:32:53 -0400 Subject: [PATCH 53/56] Version 9.4.5.0 updated for release --- VERSION | 2 +- core/pom.xml | 4 ++-- lib/pom.xml | 4 ++-- pom.xml | 5 ++++- shaded/pom.xml | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/VERSION b/VERSION index 71a0fc5c5a4..5511636bdb5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.4.5.0-SNAPSHOT +9.4.5.0 diff --git a/core/pom.xml b/core/pom.xml index faabf1e0e12..b471010dff4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -12,7 +12,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0-SNAPSHOT + 9.4.5.0 jruby-base JRuby Base @@ -711,7 +711,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-base - 9.4.5.0-SNAPSHOT + 9.4.5.0 diff --git a/lib/pom.xml b/lib/pom.xml index 1e324ccb65a..fde293698ea 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -12,7 +12,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0-SNAPSHOT + 9.4.5.0 jruby-stdlib JRuby Lib Setup @@ -28,7 +28,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-core - 9.4.5.0-SNAPSHOT + 9.4.5.0 test diff --git a/pom.xml b/pom.xml index 613d7046828..7bb97396c4d 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0-SNAPSHOT + 9.4.5.0 pom JRuby JRuby is the effort to recreate the Ruby (https://www.ruby-lang.org) interpreter in Java. @@ -274,6 +274,9 @@ DO NOT MODIFY - GENERATED CODE [3.3.0,) + + No Snapshots Allowed! + diff --git a/shaded/pom.xml b/shaded/pom.xml index dc8e2c85502..bb9d83a35b9 100644 --- a/shaded/pom.xml +++ b/shaded/pom.xml @@ -12,7 +12,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0-SNAPSHOT + 9.4.5.0 jruby-core JRuby Core From 183c04e8e4adf298051e9580e0a9402315a9fd80 Mon Sep 17 00:00:00 2001 From: "Thomas E. Enebo" Date: Thu, 2 Nov 2023 13:03:11 -0400 Subject: [PATCH 54/56] Add some sanity checks --- tool/release.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tool/release.sh b/tool/release.sh index b9b80f08675..4757da2f467 100644 --- a/tool/release.sh +++ b/tool/release.sh @@ -4,11 +4,22 @@ # 1. on failed run you must undo the version commit and probably should drop in nexus staging repo # 2. Assumes jruby you are using in your PATH works -set -x export REPO=${REPO:=jruby} -export JRUBY_VERSION=${JRUBY_VERSION:=9.4.5.0} + +java_version=$(java -version 2>&1 | head -1 | awk -F\" -e '{ print $2 }' | awk -F. '{print $1 "." $2 }') + +if [ "$java_version" != "1.8" ]; then + echo "You must use Java 1.8 to release JRuby" + exit 1 +fi + +[[ -z "$JRUBY_VERSION" ]] && { echo "set JRUBY_VERSION to something" ; exit 1; } + export WINDOWS_VERSION=`echo $JRUBY_VERSION|sed -e 's/\./_/g'` echo $JRUBY_VERSION > VERSION + +set -x + mvn git add VERSION core/pom.xml lib/pom.xml pom.xml shaded/pom.xml git commit -m "Version $JRUBY_VERSION updated for release" From 708452e56b51c9b525af1cff4e0436fb05c138bc Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 2 Nov 2023 12:39:40 -0500 Subject: [PATCH 55/56] Bump version to 9.4.6.0-SNAPSHOT --- VERSION | 2 +- core/pom.xml | 4 ++-- lib/pom.xml | 4 ++-- pom.xml | 5 +---- shaded/pom.xml | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/VERSION b/VERSION index 5511636bdb5..4d1ad8c631a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9.4.5.0 +9.4.6.0-SNAPSHOT diff --git a/core/pom.xml b/core/pom.xml index b471010dff4..3160e0694fa 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -12,7 +12,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0 + 9.4.6.0-SNAPSHOT jruby-base JRuby Base @@ -711,7 +711,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-base - 9.4.5.0 + 9.4.6.0-SNAPSHOT diff --git a/lib/pom.xml b/lib/pom.xml index fde293698ea..4ec8d384366 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -12,7 +12,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0 + 9.4.6.0-SNAPSHOT jruby-stdlib JRuby Lib Setup @@ -28,7 +28,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-core - 9.4.5.0 + 9.4.6.0-SNAPSHOT test diff --git a/pom.xml b/pom.xml index 7bb97396c4d..fcf2be8a141 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0 + 9.4.6.0-SNAPSHOT pom JRuby JRuby is the effort to recreate the Ruby (https://www.ruby-lang.org) interpreter in Java. @@ -274,9 +274,6 @@ DO NOT MODIFY - GENERATED CODE [3.3.0,) - - No Snapshots Allowed! - diff --git a/shaded/pom.xml b/shaded/pom.xml index bb9d83a35b9..4ed7d106a02 100644 --- a/shaded/pom.xml +++ b/shaded/pom.xml @@ -12,7 +12,7 @@ DO NOT MODIFY - GENERATED CODE org.jruby jruby-parent - 9.4.5.0 + 9.4.6.0-SNAPSHOT jruby-core JRuby Core From c0298f55b17731e732016a2223d3386e35b81330 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Thu, 2 Nov 2023 13:14:28 -0500 Subject: [PATCH 56/56] Clean up rubyspec tags --- spec/tags/ruby/command_line/dash_upper_i_tags.txt | 2 -- spec/tags/ruby/core/argf/each_codepoint_tags.txt | 1 - spec/tags/ruby/core/array/initialize_tags.txt | 1 - spec/tags/ruby/core/exception/no_method_error_tags.txt | 1 - spec/tags/ruby/core/file/expand_path_tags.txt | 1 - spec/tags/ruby/core/file/printf_tags.txt | 1 - spec/tags/ruby/core/io/read_tags.txt | 1 - spec/tags/ruby/core/kernel/lambda_tags.txt | 1 - spec/tags/ruby/core/kernel/printf_tags.txt | 2 -- spec/tags/ruby/core/kernel/sprintf_tags.txt | 2 -- spec/tags/ruby/core/marshal/dump_tags.txt | 3 --- spec/tags/ruby/core/module/define_method_tags.txt | 3 +-- spec/tags/ruby/core/module/instance_method_tags.txt | 1 - spec/tags/ruby/core/module/private_tags.txt | 1 - spec/tags/ruby/core/module/refine_tags.txt | 7 ------- spec/tags/ruby/core/module/ruby2_keywords_tags.txt | 2 +- spec/tags/ruby/core/queue/initialize_tags.txt | 1 - spec/tags/ruby/core/refinement/import_methods_tags.txt | 1 - spec/tags/ruby/core/regexp/encoding_tags.txt | 1 - spec/tags/ruby/core/signal/trap_tags.txt | 1 - spec/tags/ruby/core/string/lines_tags.txt | 2 -- spec/tags/ruby/core/string/modulo_tags.txt | 1 - spec/tags/ruby/core/string/undump_tags.txt | 1 - spec/tags/ruby/core/symbol/end_with_tags.txt | 2 +- spec/tags/ruby/core/thread/backtrace_tags.txt | 1 - spec/tags/ruby/core/thread/report_on_exception_tags.txt | 1 - spec/tags/ruby/core/thread/terminate_tags.txt | 2 +- spec/tags/ruby/core/unboundmethod/hash_tags.txt | 1 - spec/tags/ruby/core/warning/element_set_tags.txt | 1 - spec/tags/ruby/core/warning/warn_tags.txt | 2 -- spec/tags/ruby/language/method_tags.txt | 1 - spec/tags/ruby/language/predefined_tags.txt | 2 -- spec/tags/ruby/language/range_tags.txt | 2 +- spec/tags/ruby/language/regexp/repetition_tags.txt | 2 -- spec/tags/ruby/library/fiber/current_tags.txt | 1 - spec/tags/ruby/library/logger/logger/new_tags.txt | 1 - spec/tags/ruby/library/openstruct/to_h_tags.txt | 5 ----- spec/tags/ruby/library/stringio/printf_tags.txt | 1 - spec/tags/ruby/library/weakref/weakref_alive_tags.txt | 1 - spec/tags/ruby/library/yaml/to_yaml_tags.txt | 1 - spec/tags/ruby/library/zlib/deflate/deflate_tags.txt | 1 - spec/tags/ruby/library/zlib/inflate/inflate_tags.txt | 1 - 42 files changed, 5 insertions(+), 62 deletions(-) delete mode 100644 spec/tags/ruby/core/module/instance_method_tags.txt delete mode 100644 spec/tags/ruby/core/queue/initialize_tags.txt delete mode 100644 spec/tags/ruby/core/regexp/encoding_tags.txt delete mode 100644 spec/tags/ruby/core/string/lines_tags.txt delete mode 100644 spec/tags/ruby/core/string/undump_tags.txt delete mode 100644 spec/tags/ruby/core/unboundmethod/hash_tags.txt delete mode 100644 spec/tags/ruby/core/warning/element_set_tags.txt delete mode 100644 spec/tags/ruby/core/warning/warn_tags.txt delete mode 100644 spec/tags/ruby/language/regexp/repetition_tags.txt delete mode 100644 spec/tags/ruby/library/openstruct/to_h_tags.txt delete mode 100644 spec/tags/ruby/library/weakref/weakref_alive_tags.txt diff --git a/spec/tags/ruby/command_line/dash_upper_i_tags.txt b/spec/tags/ruby/command_line/dash_upper_i_tags.txt index 20db6ac8d0a..88cbc3c1b0b 100644 --- a/spec/tags/ruby/command_line/dash_upper_i_tags.txt +++ b/spec/tags/ruby/command_line/dash_upper_i_tags.txt @@ -1,3 +1 @@ critical(fails on Travis):The -I command line option adds the path at the front of $LOAD_PATH -fails:The -I command line option adds the path expanded from CWD to $LOAD_PATH -fails:The -I command line option expands a path from CWD even if it does not exist diff --git a/spec/tags/ruby/core/argf/each_codepoint_tags.txt b/spec/tags/ruby/core/argf/each_codepoint_tags.txt index 628660b3465..de22fb505f6 100644 --- a/spec/tags/ruby/core/argf/each_codepoint_tags.txt +++ b/spec/tags/ruby/core/argf/each_codepoint_tags.txt @@ -1,2 +1 @@ hangs:ARGF.each_codepoint returns self when passed a block -fails:ARGF.each_codepoint yields each codepoint of all streams diff --git a/spec/tags/ruby/core/array/initialize_tags.txt b/spec/tags/ruby/core/array/initialize_tags.txt index 69278a5a61a..117732388f6 100644 --- a/spec/tags/ruby/core/array/initialize_tags.txt +++ b/spec/tags/ruby/core/array/initialize_tags.txt @@ -1,2 +1 @@ fails:Array#initialize with (size, object=nil) raises an ArgumentError if size is too large -fails:Array#initialize with no arguments does not use the given block diff --git a/spec/tags/ruby/core/exception/no_method_error_tags.txt b/spec/tags/ruby/core/exception/no_method_error_tags.txt index 653a5a20886..d44ee71de18 100644 --- a/spec/tags/ruby/core/exception/no_method_error_tags.txt +++ b/spec/tags/ruby/core/exception/no_method_error_tags.txt @@ -1,2 +1 @@ fails:NoMethodError#dup copies the name, arguments and receiver -fails:NoMethodError#message uses #name to display the receiver if it is a class or a module diff --git a/spec/tags/ruby/core/file/expand_path_tags.txt b/spec/tags/ruby/core/file/expand_path_tags.txt index dc68e78b9e2..0511481f040 100644 --- a/spec/tags/ruby/core/file/expand_path_tags.txt +++ b/spec/tags/ruby/core/file/expand_path_tags.txt @@ -2,4 +2,3 @@ fails:File.expand_path raises an Encoding::CompatibilityError if the external en windows:File.expand_path does not modify a HOME string argument fails:File.expand_path when HOME is not set uses the user database when passed '~' if HOME is nil fails:File.expand_path when HOME is not set uses the user database when passed '~/' if HOME is nil -fails:File.expand_path expands a path when the default external encoding is BINARY diff --git a/spec/tags/ruby/core/file/printf_tags.txt b/spec/tags/ruby/core/file/printf_tags.txt index a4313055836..ef3eb5fc84a 100644 --- a/spec/tags/ruby/core/file/printf_tags.txt +++ b/spec/tags/ruby/core/file/printf_tags.txt @@ -24,5 +24,4 @@ fails:File#printf width specifies the minimum number of characters that will be fails:File#printf precision float types controls the number of decimal places displayed in fraction part fails:File#printf reference by name %{name} style supports flags, width and precision fails:File#printf other formats c raises TypeError if argument is nil -fails:File#printf other formats c raises TypeError if converting to String with to_str returns non-String fails:File#printf other formats c raises TypeError if converting to Integer with to_int returns non-Integer diff --git a/spec/tags/ruby/core/io/read_tags.txt b/spec/tags/ruby/core/io/read_tags.txt index 35ddb8e4e5d..00e6cc06b17 100644 --- a/spec/tags/ruby/core/io/read_tags.txt +++ b/spec/tags/ruby/core/io/read_tags.txt @@ -5,5 +5,4 @@ windows:IO#read expands the buffer when too small windows:IO#read overwrites the buffer windows:IO#read truncates the buffer when too big windows:IO#read on Windows normalizes line endings in text mode -fails:IO#read raises IOError when stream is closed by another thread fails:IO.read accepts options as keyword arguments diff --git a/spec/tags/ruby/core/kernel/lambda_tags.txt b/spec/tags/ruby/core/kernel/lambda_tags.txt index 5badd4703b5..ab3c7010a2a 100644 --- a/spec/tags/ruby/core/kernel/lambda_tags.txt +++ b/spec/tags/ruby/core/kernel/lambda_tags.txt @@ -1,2 +1 @@ fails:Kernel.lambda does not create lambda-style Procs when captured with #method -fails:Kernel.lambda when called without a literal block warns when proc isn't a lambda diff --git a/spec/tags/ruby/core/kernel/printf_tags.txt b/spec/tags/ruby/core/kernel/printf_tags.txt index dc14f19f9f9..c8bf5f371a3 100644 --- a/spec/tags/ruby/core/kernel/printf_tags.txt +++ b/spec/tags/ruby/core/kernel/printf_tags.txt @@ -49,8 +49,6 @@ fails:Kernel.printf formatting io is not specified width specifies the minimum n fails:Kernel.printf formatting io is not specified precision float types controls the number of decimal places displayed in fraction part fails:Kernel.printf formatting io is not specified reference by name %{name} style supports flags, width and precision fails:Kernel.printf formatting io is specified other formats c raises TypeError if argument is nil -fails:Kernel.printf formatting io is specified other formats c raises TypeError if converting to String with to_str returns non-String fails:Kernel.printf formatting io is specified other formats c raises TypeError if converting to Integer with to_int returns non-Integer fails:Kernel.printf formatting io is not specified other formats c raises TypeError if argument is nil -fails:Kernel.printf formatting io is not specified other formats c raises TypeError if converting to String with to_str returns non-String fails:Kernel.printf formatting io is not specified other formats c raises TypeError if converting to Integer with to_int returns non-Integer diff --git a/spec/tags/ruby/core/kernel/sprintf_tags.txt b/spec/tags/ruby/core/kernel/sprintf_tags.txt index a7b30427cb8..f188e6a0548 100644 --- a/spec/tags/ruby/core/kernel/sprintf_tags.txt +++ b/spec/tags/ruby/core/kernel/sprintf_tags.txt @@ -50,10 +50,8 @@ fails:Kernel.sprintf width specifies the minimum number of characters that will fails:Kernel.sprintf precision float types controls the number of decimal places displayed in fraction part fails:Kernel.sprintf reference by name %{name} style supports flags, width and precision fails:Kernel#sprintf other formats c raises TypeError if argument is nil -fails:Kernel#sprintf other formats c raises TypeError if converting to String with to_str returns non-String fails:Kernel#sprintf other formats c raises TypeError if converting to Integer with to_int returns non-Integer fails:Kernel.sprintf other formats c raises TypeError if argument is nil -fails:Kernel.sprintf other formats c raises TypeError if converting to String with to_str returns non-String fails:Kernel.sprintf other formats c raises TypeError if converting to Integer with to_int returns non-Integer fails:Kernel#sprintf %c raises error when a codepoint isn't representable in an encoding of a format string fails:Kernel.sprintf %c raises error when a codepoint isn't representable in an encoding of a format string diff --git a/spec/tags/ruby/core/marshal/dump_tags.txt b/spec/tags/ruby/core/marshal/dump_tags.txt index a054fffe222..a1ecbee8cb4 100644 --- a/spec/tags/ruby/core/marshal/dump_tags.txt +++ b/spec/tags/ruby/core/marshal/dump_tags.txt @@ -1,8 +1,5 @@ fails:Marshal.dump with a Time dumps the zone and the offset -fails(travis):Marshal.dump with an Exception dumps the message for the exception fails:Marshal.dump with a Symbol dumps a binary encoded Symbol -fails:Marshal.dump with an Exception dumps an empty Exception -fails:Marshal.dump with an Exception contains the filename in the backtrace fails:Marshal.dump with an Object dumps an Object with a non-US-ASCII instance variable fails:Marshal.dump with an Exception dumps the cause for the exception fails:Marshal.dump with a Time dumps the zone, but not the offset if zone is UTC diff --git a/spec/tags/ruby/core/module/define_method_tags.txt b/spec/tags/ruby/core/module/define_method_tags.txt index 8e4ff27a6d4..6b77b22328e 100644 --- a/spec/tags/ruby/core/module/define_method_tags.txt +++ b/spec/tags/ruby/core/module/define_method_tags.txt @@ -1,8 +1,7 @@ fails:Method#define_method when passed a Proc object and a method is defined inside defines the nested method in the default definee where the Proc was created -fails:Module#define_method when the default definee is not the same as the module sets the visibility of the method to public +fails(on GHA):Module#define_method when the default definee is not the same as the module sets the visibility of the method to public fails:Method#define_method when passed a block behaves exactly like a lambda for break fails:Method#define_method when passed a block behaves exactly like a lambda for next -fails:Module#define_method raises TypeError when #to_str called on non-String name returns non-String value fails:Module#define_method raises a TypeError when the given method is no Method/Proc fails:Module#define_method when passed a Proc object and a method is defined inside defines the nested method in the default definee where the Proc was created fails:Module#define_method when passed a block behaves exactly like a lambda for break diff --git a/spec/tags/ruby/core/module/instance_method_tags.txt b/spec/tags/ruby/core/module/instance_method_tags.txt deleted file mode 100644 index ebfd7613fd1..00000000000 --- a/spec/tags/ruby/core/module/instance_method_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:Module#instance_method raises TypeError when passed non-String name and #to_str returns non-String value diff --git a/spec/tags/ruby/core/module/private_tags.txt b/spec/tags/ruby/core/module/private_tags.txt index 788d7a8fcb6..8b04e50291b 100644 --- a/spec/tags/ruby/core/module/private_tags.txt +++ b/spec/tags/ruby/core/module/private_tags.txt @@ -1,3 +1,2 @@ fails:Module#private without arguments does not affect method definitions when itself is inside an eval and method definitions are outside fails:Module#private without arguments within a closure sets the visibility outside the closure -fails:Module#private continues to allow a prepended module method to call super diff --git a/spec/tags/ruby/core/module/refine_tags.txt b/spec/tags/ruby/core/module/refine_tags.txt index c2ddcfe693a..f56a17a41d6 100644 --- a/spec/tags/ruby/core/module/refine_tags.txt +++ b/spec/tags/ruby/core/module/refine_tags.txt @@ -1,12 +1,5 @@ -fails:Module#refine for methods accessed indirectly is honored by Kernel#send -fails:Module#refine for methods accessed indirectly is honored by BasicObject#__send__ fails:Module#refine for methods accessed indirectly is not honored by & fails(not implemented, jruby/jruby#6161):Module#refine for methods accessed indirectly is honored by & -fails(not implemented, jruby/jruby#6161):Module#refine for methods accessed indirectly is honored by Kernel#public_send -fails(not implemented, jruby/jruby#6161):Module#refine for methods accessed indirectly is honored by Kernel#respond_to? fails:Module#refine when super is called in a refinement looks in the lexical scope refinements before other active refinements -fails:Module#refine for methods accessed indirectly is honored by Kernel#method fails:Module#refine for methods accessed indirectly is honored by Kernel#public_method -fails:Module#refine for methods accessed indirectly is honored by Kernel#instance_method -fails:Module#refine applies refinements to calls in the refine block fails:Module#refine raises TypeError if not passed a class diff --git a/spec/tags/ruby/core/module/ruby2_keywords_tags.txt b/spec/tags/ruby/core/module/ruby2_keywords_tags.txt index 26967858ec1..b6df80f81b5 100644 --- a/spec/tags/ruby/core/module/ruby2_keywords_tags.txt +++ b/spec/tags/ruby/core/module/ruby2_keywords_tags.txt @@ -1,2 +1,2 @@ fails:Module#ruby2_keywords does NOT copy the Hash when calling a method taking (*args) -fails:Module#ruby2_keywords makes a copy and unmark the Hash when calling a method taking (arg) +fails(JIT mode only):Module#ruby2_keywords makes a copy and unmark the Hash when calling a method taking (arg) diff --git a/spec/tags/ruby/core/queue/initialize_tags.txt b/spec/tags/ruby/core/queue/initialize_tags.txt deleted file mode 100644 index a8d2cf8fea8..00000000000 --- a/spec/tags/ruby/core/queue/initialize_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:Queue#initialize raises TypeError if #to_a does not return Array diff --git a/spec/tags/ruby/core/refinement/import_methods_tags.txt b/spec/tags/ruby/core/refinement/import_methods_tags.txt index a7e03894c00..5f71d530cde 100644 --- a/spec/tags/ruby/core/refinement/import_methods_tags.txt +++ b/spec/tags/ruby/core/refinement/import_methods_tags.txt @@ -1,4 +1,3 @@ -fails:Refinement#import_methods when methods are defined in Ruby code imports methods fails:Refinement#import_methods doesn't import any methods if one of the arguments is not a module fails:Refinement#import_methods imports methods from multiple modules so that methods see other's module's methods fails:Refinement#import_methods imports methods from module so that methods can see each other diff --git a/spec/tags/ruby/core/regexp/encoding_tags.txt b/spec/tags/ruby/core/regexp/encoding_tags.txt deleted file mode 100644 index fc74e8f486b..00000000000 --- a/spec/tags/ruby/core/regexp/encoding_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:Regexp#encoding allows otherwise invalid characters if NOENCODING is specified diff --git a/spec/tags/ruby/core/signal/trap_tags.txt b/spec/tags/ruby/core/signal/trap_tags.txt index 09d157adbd8..8dce29baa9d 100644 --- a/spec/tags/ruby/core/signal/trap_tags.txt +++ b/spec/tags/ruby/core/signal/trap_tags.txt @@ -7,5 +7,4 @@ fails:Signal.trap raises ArgumentError or Errno::EINVAL for SIGSTOP fails:Signal.trap accepts 'SYSTEM_DEFAULT' and uses the OS handler for SIGPIPE fails:Signal.trap allows to register a handler for all known signals, except reserved signals for which it raises ArgumentError fails:Signal.trap raises ArgumentError when passed unknown signal - fails:Signal.trap calls #to_str on an object to convert to a String diff --git a/spec/tags/ruby/core/string/lines_tags.txt b/spec/tags/ruby/core/string/lines_tags.txt deleted file mode 100644 index 3b3806dc9bf..00000000000 --- a/spec/tags/ruby/core/string/lines_tags.txt +++ /dev/null @@ -1,2 +0,0 @@ -fails:String#lines when `chomp` keyword argument is passed removes only specified separator -fails:String#lines when `chomp` keyword argument is passed ignores new line characters when separator is specified diff --git a/spec/tags/ruby/core/string/modulo_tags.txt b/spec/tags/ruby/core/string/modulo_tags.txt index c0b21505b93..6fe3b98afea 100644 --- a/spec/tags/ruby/core/string/modulo_tags.txt +++ b/spec/tags/ruby/core/string/modulo_tags.txt @@ -24,6 +24,5 @@ fails:String#% width specifies the minimum number of characters that will be wri fails:String#% precision float types controls the number of decimal places displayed in fraction part fails:String#% reference by name %{name} style supports flags, width and precision fails:String#% other formats c raises TypeError if argument is nil -fails:String#% other formats c raises TypeError if converting to String with to_str returns non-String fails:String#% other formats c raises TypeError if converting to Integer with to_int returns non-Integer fails:String#% %c raises error when a codepoint isn't representable in an encoding of a format string diff --git a/spec/tags/ruby/core/string/undump_tags.txt b/spec/tags/ruby/core/string/undump_tags.txt deleted file mode 100644 index f3cb250489a..00000000000 --- a/spec/tags/ruby/core/string/undump_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:String#undump Limitations cannot undump non ASCII-compatible string diff --git a/spec/tags/ruby/core/symbol/end_with_tags.txt b/spec/tags/ruby/core/symbol/end_with_tags.txt index 7340b3604cc..ca4cca3c18d 100644 --- a/spec/tags/ruby/core/symbol/end_with_tags.txt +++ b/spec/tags/ruby/core/symbol/end_with_tags.txt @@ -1 +1 @@ -fails:Symbol#end_with? checks that we are starting to match at the head of a character +fails(on GHA):Symbol#end_with? checks that we are starting to match at the head of a character \ No newline at end of file diff --git a/spec/tags/ruby/core/thread/backtrace_tags.txt b/spec/tags/ruby/core/thread/backtrace_tags.txt index 251c5604367..faaf0defe67 100644 --- a/spec/tags/ruby/core/thread/backtrace_tags.txt +++ b/spec/tags/ruby/core/thread/backtrace_tags.txt @@ -1,4 +1,3 @@ fails:Thread#backtrace returns an array (which may be empty) immediately after the thread is created fails:Thread#backtrace returns the current backtrace of a thread fails:Thread#backtrace can be called with a range whose end is negative -fails:Thread#backtrace returns nil if omitting more locations than available diff --git a/spec/tags/ruby/core/thread/report_on_exception_tags.txt b/spec/tags/ruby/core/thread/report_on_exception_tags.txt index f215ff5d223..10a8e52a012 100644 --- a/spec/tags/ruby/core/thread/report_on_exception_tags.txt +++ b/spec/tags/ruby/core/thread/report_on_exception_tags.txt @@ -1,2 +1 @@ fails:Thread#report_on_exception= when set to true prints a backtrace on $stderr in the regular backtrace order -fails:Thread#report_on_exception= when set to true prints the backtrace even if the thread was killed just after Thread#raise diff --git a/spec/tags/ruby/core/thread/terminate_tags.txt b/spec/tags/ruby/core/thread/terminate_tags.txt index aedde8fc40e..deb5a1a3cea 100644 --- a/spec/tags/ruby/core/thread/terminate_tags.txt +++ b/spec/tags/ruby/core/thread/terminate_tags.txt @@ -1,2 +1,2 @@ unstable:Thread#terminate killing dying running does nothing -fails(we disagree that ensures should not be run):Thread#terminate kills other fibers of that thread without running their ensure clauses \ No newline at end of file +unstable(and we disagree with ruby-core about whether ensures should run):Thread#terminate kills other fibers of that thread without running their ensure clauses \ No newline at end of file diff --git a/spec/tags/ruby/core/unboundmethod/hash_tags.txt b/spec/tags/ruby/core/unboundmethod/hash_tags.txt deleted file mode 100644 index 033a044a20e..00000000000 --- a/spec/tags/ruby/core/unboundmethod/hash_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:UnboundMethod#hash returns the same value for builtin methods that are eql? diff --git a/spec/tags/ruby/core/warning/element_set_tags.txt b/spec/tags/ruby/core/warning/element_set_tags.txt deleted file mode 100644 index f5f4b2936a8..00000000000 --- a/spec/tags/ruby/core/warning/element_set_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails:Warning.[]= :experimental emits and suppresses warnings for :experimental diff --git a/spec/tags/ruby/core/warning/warn_tags.txt b/spec/tags/ruby/core/warning/warn_tags.txt deleted file mode 100644 index 8c45348bb34..00000000000 --- a/spec/tags/ruby/core/warning/warn_tags.txt +++ /dev/null @@ -1,2 +0,0 @@ -fails:Warning.warn is called by Kernel.warn with nil category keyword -fails:Warning.warn is called by Kernel.warn with given category keyword converted to a symbol diff --git a/spec/tags/ruby/language/method_tags.txt b/spec/tags/ruby/language/method_tags.txt index d254a6651f4..67c0e5148ad 100644 --- a/spec/tags/ruby/language/method_tags.txt +++ b/spec/tags/ruby/language/method_tags.txt @@ -1,2 +1 @@ -fails:A method assigns local variables from method parameters for definition 'def m(*a) a end' fails:A method assigns local variables from method parameters for definition 'def m(a, **nil); a end;' diff --git a/spec/tags/ruby/language/predefined_tags.txt b/spec/tags/ruby/language/predefined_tags.txt index 00debe69998..29e89f3a3b7 100644 --- a/spec/tags/ruby/language/predefined_tags.txt +++ b/spec/tags/ruby/language/predefined_tags.txt @@ -4,5 +4,3 @@ windows:The predefined global constant ARGV contains Strings encoded in locale E fails:Predefined global $! in bodies without ensure should be cleared when an exception is rescued even when a non-local return is present fails(etc not .so/.jar):$LOAD_PATH.resolve_feature_path returns what will be loaded without actual loading, .so file fails:Execution variable $: default $LOAD_PATH entries until sitelibdir included have @gem_prelude_index set -fails:Predefined global $, warns if assigned non-nil -fails:Predefined global $; warns if assigned non-nil diff --git a/spec/tags/ruby/language/range_tags.txt b/spec/tags/ruby/language/range_tags.txt index 64bc2fa65f3..d6fb003fec9 100644 --- a/spec/tags/ruby/language/range_tags.txt +++ b/spec/tags/ruby/language/range_tags.txt @@ -1 +1 @@ -fails:Literal Ranges creates a simple range as an object literal +fails(JIT mode only):Literal Ranges creates a simple range as an object literal diff --git a/spec/tags/ruby/language/regexp/repetition_tags.txt b/spec/tags/ruby/language/regexp/repetition_tags.txt deleted file mode 100644 index 348391570af..00000000000 --- a/spec/tags/ruby/language/regexp/repetition_tags.txt +++ /dev/null @@ -1,2 +0,0 @@ -fails:Regexps with repetition does not treat {m,n}+ as possessive -fails:Regexps with repetition supports nested quantifiers diff --git a/spec/tags/ruby/library/fiber/current_tags.txt b/spec/tags/ruby/library/fiber/current_tags.txt index b77dd3065e5..f83d68ad6c0 100644 --- a/spec/tags/ruby/library/fiber/current_tags.txt +++ b/spec/tags/ruby/library/fiber/current_tags.txt @@ -1,2 +1 @@ fails:Fiber.current returns the current Fiber when called from a Fiber that transferred to another -fails:Fiber.current is available without an extra require diff --git a/spec/tags/ruby/library/logger/logger/new_tags.txt b/spec/tags/ruby/library/logger/logger/new_tags.txt index 788dee7c106..1cf912d3055 100644 --- a/spec/tags/ruby/library/logger/logger/new_tags.txt +++ b/spec/tags/ruby/library/logger/logger/new_tags.txt @@ -1,2 +1 @@ windows:Logger#new creates a new logger object - diff --git a/spec/tags/ruby/library/openstruct/to_h_tags.txt b/spec/tags/ruby/library/openstruct/to_h_tags.txt deleted file mode 100644 index 04e62cf398d..00000000000 --- a/spec/tags/ruby/library/openstruct/to_h_tags.txt +++ /dev/null @@ -1,5 +0,0 @@ -fails(reverted ostruct due to https://github.com/ruby/ostruct/issues/30):OpenStruct#to_h with block converts [key, value] pairs returned by the block to a hash -fails(reverted ostruct due to https://github.com/ruby/ostruct/issues/30):OpenStruct#to_h with block raises ArgumentError if block returns longer or shorter array -fails(reverted ostruct due to https://github.com/ruby/ostruct/issues/30):OpenStruct#to_h with block raises TypeError if block returns something other than Array -fails(reverted ostruct due to https://github.com/ruby/ostruct/issues/30):OpenStruct#to_h with block coerces returned pair to Array with #to_ary -fails(reverted ostruct due to https://github.com/ruby/ostruct/issues/30):OpenStruct#to_h with block does not coerce returned pair to Array with #to_a diff --git a/spec/tags/ruby/library/stringio/printf_tags.txt b/spec/tags/ruby/library/stringio/printf_tags.txt index a0c4adbf3ef..c08afc7bf51 100644 --- a/spec/tags/ruby/library/stringio/printf_tags.txt +++ b/spec/tags/ruby/library/stringio/printf_tags.txt @@ -24,5 +24,4 @@ fails:StringIO#printf formatting width specifies the minimum number of character fails:StringIO#printf formatting precision float types controls the number of decimal places displayed in fraction part fails:StringIO#printf formatting reference by name %{name} style supports flags, width and precision fails:StringIO#printf formatting other formats c raises TypeError if argument is nil -fails:StringIO#printf formatting other formats c raises TypeError if converting to String with to_str returns non-String fails:StringIO#printf formatting other formats c raises TypeError if converting to Integer with to_int returns non-Integer diff --git a/spec/tags/ruby/library/weakref/weakref_alive_tags.txt b/spec/tags/ruby/library/weakref/weakref_alive_tags.txt deleted file mode 100644 index ee77dd8fc64..00000000000 --- a/spec/tags/ruby/library/weakref/weakref_alive_tags.txt +++ /dev/null @@ -1 +0,0 @@ -fails(depends on GC behavior, fails intermittently):WeakRef#weakref_alive? returns a falsy value if the object is no longer reachable diff --git a/spec/tags/ruby/library/yaml/to_yaml_tags.txt b/spec/tags/ruby/library/yaml/to_yaml_tags.txt index 5c489f6230e..8a2304db8fb 100644 --- a/spec/tags/ruby/library/yaml/to_yaml_tags.txt +++ b/spec/tags/ruby/library/yaml/to_yaml_tags.txt @@ -1,2 +1 @@ fails:Object#to_yaml returns the YAML representation of a RegExp object -fails:Object#to_yaml returns the YAML representation of a Error object diff --git a/spec/tags/ruby/library/zlib/deflate/deflate_tags.txt b/spec/tags/ruby/library/zlib/deflate/deflate_tags.txt index f53c089dd72..0126f382161 100644 --- a/spec/tags/ruby/library/zlib/deflate/deflate_tags.txt +++ b/spec/tags/ruby/library/zlib/deflate/deflate_tags.txt @@ -3,4 +3,3 @@ fails:Zlib::Deflate#deflate without break deflates chunked data fails:Zlib::Deflate#deflate without break deflates chunked data with final chunk fails:Zlib::Deflate#deflate with break deflates only first chunk fails:Zlib::Deflate#deflate with break deflates chunked data with final chunk -fails:Zlib::Deflate#deflate has a binary encoding diff --git a/spec/tags/ruby/library/zlib/inflate/inflate_tags.txt b/spec/tags/ruby/library/zlib/inflate/inflate_tags.txt index 43eea6a4133..2c3eb14a842 100644 --- a/spec/tags/ruby/library/zlib/inflate/inflate_tags.txt +++ b/spec/tags/ruby/library/zlib/inflate/inflate_tags.txt @@ -1,2 +1 @@ fails:Zlib::Inflate#inflate without break inflates chunked data -fails:Zlib::Inflate#inflate has a binary encoding