Allocate block-exceeding tokens directly in the old generation

include/runtime/types.h

@@ -96,7 +96,15 @@ __attribute__((always_inline)) constexpr uint64_t len(T const *s) {
 
 template <typename T>
 __attribute__((always_inline)) constexpr void init_with_len(T *s, uint64_t l) {
-  s->h.hdr = l | (l > BLOCK_SIZE - sizeof(char *) ? NOT_YOUNG_OBJECT_BIT : 0);
+  // A token larger than a block cannot live in the young semispace's per-block model, so it is
+  // marked NOT_YOUNG and (see kore_alloc_token) allocated directly in the old generation. It
+  // must also carry the AGE bit so that is_in_old_gen_hdr() recognises it as a live heap block;
+  // otherwise the collector treats it as a non-heap/static block, never migrates it, and the
+  // young from-space it was (incorrectly) placed in gets reclaimed, dangling the reference.
+  s->h.hdr
+      = l
+        | (l > BLOCK_SIZE - sizeof(char *) ? (NOT_YOUNG_OBJECT_BIT | AGE_MASK)
+                                           : 0);
 }
 
 __attribute__((always_inline)) constexpr uint64_t size_hdr(uint64_t hdr) {

runtime/lto/alloc.cpp

@@ -60,9 +60,28 @@ __attribute__((always_inline)) void *kore_alloc(size_t requested) {
   return youngspace.kore_arena_alloc(requested);
 }
 
+// Cold, out-of-line path for the rare oversized token (see kore_alloc_token). Kept out of the
+// always_inline hot path so the common small-token case is not bloated at every call site.
+namespace {
+__attribute__((noinline)) void *kore_alloc_token_oldspace(size_t size) {
+  return oldspace.kore_arena_alloc(size);
+}
+} // namespace
+
 __attribute__((always_inline)) void *kore_alloc_token(size_t requested) {
   size_t size = (requested + 7) & ~7;
-  return youngspace.kore_arena_alloc(size < 16 ? 16 : size);
+  if (size < 16) {
+    size = 16;
+  }
+  // A token whose total size exceeds a block is flagged NOT_YOUNG by init_with_len (it cannot
+  // participate in the young semispace's block model). Such tokens must be allocated directly in
+  // the old generation: the young from-space is reclaimed on every collection, and the collector
+  // never migrates a NOT_YOUNG object out of it, so a large token left in the young space would
+  // dangle. This matches the existing large-buffer handling in hook_BUFFER_concat_raw.
+  if (__builtin_expect(size > BLOCK_SIZE, 0)) {
+    return kore_alloc_token_oldspace(size);
+  }
+  return youngspace.kore_arena_alloc(size);
 }
 
 __attribute__((always_inline)) void *kore_alloc_old(size_t requested) {

test/defn/k-files/test-gc-large-token.k

@@ -0,0 +1,57 @@
+// Regression test for the large-token young-generation GC bug.
+//
+// A String token whose payload exceeds BLOCK_SIZE (1 MiB) is flagged NOT_YOUNG by
+// init_with_len, but kore_alloc_token used to place it in the young semispace anyway.
+// The collector never migrates a NOT_YOUNG object out of the young from-space (which is
+// reclaimed on every collection), so such a token dangled: a later allocation overwrote
+// its bytes and a subsequent collection walked a corrupted header.
+//
+// This definition (a) builds a single > 1 MiB String token, (b) keeps it live in the
+// <keep> cell across (c) heavy subsequent allocation that forces collections and reuses
+// the abandoned region, then (d) reads the token's length back. On the buggy runtime the
+// length is corrupted (or the run crashes); with the fix the token lives in the old
+// generation and its length survives intact.
+module TEST
+  imports STRING-BUFFER
+  imports INT
+  imports STRING
+  imports MAP
+
+  configuration <T>
+    <k> $PGM:Pgm </k>
+    <buffer> .StringBuffer </buffer>
+    <keep> "":String </keep>
+    <m> .Map </m>
+  </T>
+
+  syntax Cmd ::= appendN(Int, String)
+               | freeze()
+               | churn(Int)
+  syntax Pgm ::= List{Cmd,";"}
+
+  // Build a > 1 MiB buffer by appending S, I times.
+  rule <k> appendN(I, S) ; Cmds => appendN(I -Int 1, S) ; Cmds </k>
+       <buffer> SB => SB +String S </buffer>
+    requires I >Int 0
+  rule <k> appendN(I, _) ; Cmds => Cmds </k>
+    requires I ==Int 0
+
+  // Materialize the buffer as one > 1 MiB String token and stash it as a live GC root.
+  rule <k> freeze() ; Cmds => Cmds </k>
+       <buffer> SB </buffer>
+       <keep> _ => StringBuffer2String(SB) </keep>
+
+  // Allocate heavily to force collections after freeze() and reuse the abandoned region.
+  rule <k> churn(I) ; Cmds => churn(I -Int 1) ; Cmds </k>
+       <m> M => M [ Int2String(I) <- I ] </m>
+    requires I >Int 0
+  rule <k> churn(I) ; Cmds => Cmds </k>
+    requires I ==Int 0
+
+  // Read the kept token back: its length must have survived the collections intact.
+  // Clear the bulky cells so the checked output stays small.
+  rule <k> .Pgm => lengthString(S) </k>
+       <keep> S => "" </keep>
+       <buffer> _ => .StringBuffer </buffer>
+       <m> _ => .Map </m>
+endmodule