{ "rows": [ { "encoding": "cl100k_base", "pieces": [ { "bytes_hex": "66726f6d", "token_id": 1527, "utf8_fragment": "from" }, { "bytes_hex": "20646563696d616c", "token_id": 12395, "utf8_fragment": " decimal" }, { "bytes_hex": "20696d706f7274", "token_id": 1179, "utf8_fragment": " import" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "0a0a0a", "token_id": 1432, "utf8_fragment": "\n\n\n" }, { "bytes_hex": "4d", "token_id": 44, "utf8_fragment": "M" }, { "bytes_hex": "494c4c", "token_id": 9410, "utf8_fragment": "ILL" }, { "bytes_hex": "494f4e", "token_id": 1294, "utf8_fragment": "ION" }, { "bytes_hex": "203d", "token_id": 284, "utf8_fragment": " =" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "2822", "token_id": 446, "utf8_fragment": "(\"" }, { "bytes_hex": "313030", "token_id": 1041, "utf8_fragment": "100" }, { "bytes_hex": "303030", "token_id": 931, "utf8_fragment": "000" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "22290a0a0a", "token_id": 26722, "utf8_fragment": "\")\n\n\n" }, { "bytes_hex": "646566", "token_id": 755, "utf8_fragment": "def" }, { "bytes_hex": "206d6f6e74686c79", "token_id": 15438, "utf8_fragment": " monthly" }, { "bytes_hex": "5f636f7374", "token_id": 16269, "utf8_fragment": "_cost" }, { "bytes_hex": "280a", "token_id": 1021, "utf8_fragment": "(\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "207265717565737473", "token_id": 7540, "utf8_fragment": " requests" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 528, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 528, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20636163686564", "token_id": 21224, "utf8_fragment": " cached" }, { "bytes_hex": "5f696e707574", "token_id": 6022, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 528, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "206f7574707574", "token_id": 2612, "utf8_fragment": " output" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 528, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "5f72617465", "token_id": 9430, "utf8_fragment": "_rate" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20636163686564", "token_id": 21224, "utf8_fragment": " cached" }, { "bytes_hex": "5f72617465", "token_id": 9430, "utf8_fragment": "_rate" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "206f7574707574", "token_id": 2612, "utf8_fragment": " output" }, { "bytes_hex": "5f72617465", "token_id": 9430, "utf8_fragment": "_rate" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "2c0a", "token_id": 345, "utf8_fragment": ",\n" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202d3e", "token_id": 1492, "utf8_fragment": " ->" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "3a0a", "token_id": 512, "utf8_fragment": ":\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20222222", "token_id": 4304, "utf8_fragment": " \"\"\"" }, { "bytes_hex": "457374", "token_id": 14101, "utf8_fragment": "Est" }, { "bytes_hex": "696d617465", "token_id": 3509, "utf8_fragment": "imate" }, { "bytes_hex": "2061", "token_id": 264, "utf8_fragment": " a" }, { "bytes_hex": "206d6f6e74686c79", "token_id": 15438, "utf8_fragment": " monthly" }, { "bytes_hex": "20746f6b656e", "token_id": 4037, "utf8_fragment": " token" }, { "bytes_hex": "20636861726765", "token_id": 6900, "utf8_fragment": " charge" }, { "bytes_hex": "2066726f6d", "token_id": 505, "utf8_fragment": " from" }, { "bytes_hex": "206578706c69636974", "token_id": 11720, "utf8_fragment": " explicit" }, { "bytes_hex": "207261746573", "token_id": 7969, "utf8_fragment": " rates" }, { "bytes_hex": "2e2222220a", "token_id": 14781, "utf8_fragment": ".\"\"\"\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "206966", "token_id": 422, "utf8_fragment": " if" }, { "bytes_hex": "206e6f74", "token_id": 539, "utf8_fragment": " not" }, { "bytes_hex": "20", "token_id": 220, "utf8_fragment": " " }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "203c3d", "token_id": 2717, "utf8_fragment": " <=" }, { "bytes_hex": "20636163686564", "token_id": 21224, "utf8_fragment": " cached" }, { "bytes_hex": "5f696e707574", "token_id": 6022, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "203c3d", "token_id": 2717, "utf8_fragment": " <=" }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a0a", "token_id": 512, "utf8_fragment": ":\n" }, { "bytes_hex": "20202020202020", "token_id": 286, "utf8_fragment": " " }, { "bytes_hex": "207261697365", "token_id": 4933, "utf8_fragment": " raise" }, { "bytes_hex": "2056616c75654572726f72", "token_id": 15764, "utf8_fragment": " ValueError" }, { "bytes_hex": "2822", "token_id": 446, "utf8_fragment": "(\"" }, { "bytes_hex": "636163686564", "token_id": 34018, "utf8_fragment": "cached" }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "206d757374", "token_id": 2011, "utf8_fragment": " must" }, { "bytes_hex": "206265", "token_id": 387, "utf8_fragment": " be" }, { "bytes_hex": "2061", "token_id": 264, "utf8_fragment": " a" }, { "bytes_hex": "20737562736574", "token_id": 27084, "utf8_fragment": " subset" }, { "bytes_hex": "206f66", "token_id": 315, "utf8_fragment": " of" }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "22290a0a", "token_id": 5240, "utf8_fragment": "\")\n\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20756e63", "token_id": 21482, "utf8_fragment": " unc" }, { "bytes_hex": "6163686564", "token_id": 3939, "utf8_fragment": "ached" }, { "bytes_hex": "203d", "token_id": 284, "utf8_fragment": " =" }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "202d", "token_id": 482, "utf8_fragment": " -" }, { "bytes_hex": "20636163686564", "token_id": 21224, "utf8_fragment": " cached" }, { "bytes_hex": "5f696e707574", "token_id": 6022, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "20706572", "token_id": 824, "utf8_fragment": " per" }, { "bytes_hex": "5f72657175657374", "token_id": 8052, "utf8_fragment": "_request" }, { "bytes_hex": "203d", "token_id": 284, "utf8_fragment": " =" }, { "bytes_hex": "20280a", "token_id": 2456, "utf8_fragment": " (\n" }, { "bytes_hex": "20202020202020", "token_id": 286, "utf8_fragment": " " }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "28", "token_id": 7, "utf8_fragment": "(" }, { "bytes_hex": "756e63", "token_id": 1371, "utf8_fragment": "unc" }, { "bytes_hex": "6163686564", "token_id": 3939, "utf8_fragment": "ached" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 353, "utf8_fragment": " *" }, { "bytes_hex": "20696e707574", "token_id": 1988, "utf8_fragment": " input" }, { "bytes_hex": "5f72617465", "token_id": 9430, "utf8_fragment": "_rate" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "20202020202020", "token_id": 286, "utf8_fragment": " " }, { "bytes_hex": "202b", "token_id": 489, "utf8_fragment": " +" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "2863", "token_id": 1361, "utf8_fragment": "(c" }, { "bytes_hex": "6163686564", "token_id": 3939, "utf8_fragment": "ached" }, { "bytes_hex": "5f696e707574", "token_id": 6022, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 353, "utf8_fragment": " *" }, { "bytes_hex": "20636163686564", "token_id": 21224, "utf8_fragment": " cached" }, { "bytes_hex": "5f72617465", "token_id": 9430, "utf8_fragment": "_rate" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "20202020202020", "token_id": 286, "utf8_fragment": " " }, { "bytes_hex": "202b", "token_id": 489, "utf8_fragment": " +" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "286f7574707574", "token_id": 11304, "utf8_fragment": "(output" }, { "bytes_hex": "5f746f6b656e73", "token_id": 29938, "utf8_fragment": "_tokens" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 353, "utf8_fragment": " *" }, { "bytes_hex": "206f7574707574", "token_id": 2612, "utf8_fragment": " output" }, { "bytes_hex": "5f72617465", "token_id": 9430, "utf8_fragment": "_rate" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "2029", "token_id": 883, "utf8_fragment": " )" }, { "bytes_hex": "202f", "token_id": 611, "utf8_fragment": " /" }, { "bytes_hex": "204d494c4c", "token_id": 83585, "utf8_fragment": " MILL" }, { "bytes_hex": "494f4e", "token_id": 1294, "utf8_fragment": "ION" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "202020", "token_id": 262, "utf8_fragment": " " }, { "bytes_hex": "2072657475726e", "token_id": 471, "utf8_fragment": " return" }, { "bytes_hex": "20446563696d616c", "token_id": 27826, "utf8_fragment": " Decimal" }, { "bytes_hex": "2872657175657374", "token_id": 4980, "utf8_fragment": "(request" }, { "bytes_hex": "73", "token_id": 82, "utf8_fragment": "s" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 353, "utf8_fragment": " *" }, { "bytes_hex": "20706572", "token_id": 824, "utf8_fragment": " per" }, { "bytes_hex": "5f72657175657374", "token_id": 8052, "utf8_fragment": "_request" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" } ], "sample_id": "codigo", "source_sha256": "e576659845158bb30dc9d8443970e284418f7308c2df3ef4b91966aa14f043d0", "token_ids": [ 1527, 12395, 1179, 27826, 1432, 44, 9410, 1294, 284, 27826, 446, 1041, 931, 15, 26722, 755, 15438, 16269, 1021, 262, 7540, 25, 528, 345, 262, 1988, 29938, 25, 528, 345, 262, 21224, 6022, 29938, 25, 528, 345, 262, 2612, 29938, 25, 528, 345, 262, 1988, 9430, 25, 27826, 345, 262, 21224, 9430, 25, 27826, 345, 262, 2612, 9430, 25, 27826, 345, 8, 1492, 27826, 512, 262, 4304, 14101, 3509, 264, 15438, 4037, 6900, 505, 11720, 7969, 14781, 262, 422, 539, 220, 15, 2717, 21224, 6022, 29938, 2717, 1988, 29938, 512, 286, 4933, 15764, 446, 34018, 1988, 2011, 387, 264, 27084, 315, 1988, 5240, 262, 21482, 3939, 284, 1988, 29938, 482, 21224, 6022, 29938, 198, 262, 824, 8052, 284, 2456, 286, 27826, 7, 1371, 3939, 8, 353, 1988, 9430, 198, 286, 489, 27826, 1361, 3939, 6022, 29938, 8, 353, 21224, 9430, 198, 286, 489, 27826, 11304, 29938, 8, 353, 2612, 9430, 198, 262, 883, 611, 83585, 1294, 198, 262, 471, 27826, 4980, 82, 8, 353, 824, 8052, 198 ] }, { "encoding": "o200k_base", "pieces": [ { "bytes_hex": "66726f6d", "token_id": 2845, "utf8_fragment": "from" }, { "bytes_hex": "20646563696d616c", "token_id": 26964, "utf8_fragment": " decimal" }, { "bytes_hex": "20696d706f7274", "token_id": 1588, "utf8_fragment": " import" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "0a0a0a", "token_id": 2499, "utf8_fragment": "\n\n\n" }, { "bytes_hex": "4d", "token_id": 44, "utf8_fragment": "M" }, { "bytes_hex": "494c4c", "token_id": 16050, "utf8_fragment": "ILL" }, { "bytes_hex": "494f4e", "token_id": 2044, "utf8_fragment": "ION" }, { "bytes_hex": "203d", "token_id": 314, "utf8_fragment": " =" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "2822", "token_id": 568, "utf8_fragment": "(\"" }, { "bytes_hex": "313030", "token_id": 1353, "utf8_fragment": "100" }, { "bytes_hex": "303030", "token_id": 1302, "utf8_fragment": "000" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "22290a0a0a", "token_id": 57903, "utf8_fragment": "\")\n\n\n" }, { "bytes_hex": "646566", "token_id": 1314, "utf8_fragment": "def" }, { "bytes_hex": "206d6f6e74686c79", "token_id": 16213, "utf8_fragment": " monthly" }, { "bytes_hex": "5f636f7374", "token_id": 36820, "utf8_fragment": "_cost" }, { "bytes_hex": "280a", "token_id": 1607, "utf8_fragment": "(\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "207265717565737473", "token_id": 13782, "utf8_fragment": " requests" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 677, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 677, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20636163686564", "token_id": 39472, "utf8_fragment": " cached" }, { "bytes_hex": "5f696e707574", "token_id": 12507, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 677, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "206f7574707574", "token_id": 4733, "utf8_fragment": " output" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20696e74", "token_id": 677, "utf8_fragment": " int" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "5f72617465", "token_id": 20477, "utf8_fragment": "_rate" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20636163686564", "token_id": 39472, "utf8_fragment": " cached" }, { "bytes_hex": "5f72617465", "token_id": 20477, "utf8_fragment": "_rate" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "206f7574707574", "token_id": 4733, "utf8_fragment": " output" }, { "bytes_hex": "5f72617465", "token_id": 20477, "utf8_fragment": "_rate" }, { "bytes_hex": "3a", "token_id": 25, "utf8_fragment": ":" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "2c0a", "token_id": 412, "utf8_fragment": ",\n" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202d3e", "token_id": 2747, "utf8_fragment": " ->" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "3a0a", "token_id": 734, "utf8_fragment": ":\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20222222", "token_id": 9280, "utf8_fragment": " \"\"\"" }, { "bytes_hex": "457374696d617465", "token_id": 96726, "utf8_fragment": "Estimate" }, { "bytes_hex": "2061", "token_id": 261, "utf8_fragment": " a" }, { "bytes_hex": "206d6f6e74686c79", "token_id": 16213, "utf8_fragment": " monthly" }, { "bytes_hex": "20746f6b656e", "token_id": 6602, "utf8_fragment": " token" }, { "bytes_hex": "20636861726765", "token_id": 8363, "utf8_fragment": " charge" }, { "bytes_hex": "2066726f6d", "token_id": 591, "utf8_fragment": " from" }, { "bytes_hex": "206578706c69636974", "token_id": 23141, "utf8_fragment": " explicit" }, { "bytes_hex": "207261746573", "token_id": 9837, "utf8_fragment": " rates" }, { "bytes_hex": "2e2222220a", "token_id": 33044, "utf8_fragment": ".\"\"\"\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "206966", "token_id": 538, "utf8_fragment": " if" }, { "bytes_hex": "206e6f74", "token_id": 625, "utf8_fragment": " not" }, { "bytes_hex": "20", "token_id": 220, "utf8_fragment": " " }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "203c3d", "token_id": 5017, "utf8_fragment": " <=" }, { "bytes_hex": "20636163686564", "token_id": 39472, "utf8_fragment": " cached" }, { "bytes_hex": "5f696e707574", "token_id": 12507, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "203c3d", "token_id": 5017, "utf8_fragment": " <=" }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "3a0a", "token_id": 734, "utf8_fragment": ":\n" }, { "bytes_hex": "20202020202020", "token_id": 309, "utf8_fragment": " " }, { "bytes_hex": "207261697365", "token_id": 9338, "utf8_fragment": " raise" }, { "bytes_hex": "2056616c7565", "token_id": 9963, "utf8_fragment": " Value" }, { "bytes_hex": "4572726f72", "token_id": 2255, "utf8_fragment": "Error" }, { "bytes_hex": "2822", "token_id": 568, "utf8_fragment": "(\"" }, { "bytes_hex": "636163686564", "token_id": 62023, "utf8_fragment": "cached" }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "206d757374", "token_id": 2804, "utf8_fragment": " must" }, { "bytes_hex": "206265", "token_id": 413, "utf8_fragment": " be" }, { "bytes_hex": "2061", "token_id": 261, "utf8_fragment": " a" }, { "bytes_hex": "20737562736574", "token_id": 58845, "utf8_fragment": " subset" }, { "bytes_hex": "206f66", "token_id": 328, "utf8_fragment": " of" }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "22290a0a", "token_id": 10822, "utf8_fragment": "\")\n\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20756e63", "token_id": 34591, "utf8_fragment": " unc" }, { "bytes_hex": "6163686564", "token_id": 5908, "utf8_fragment": "ached" }, { "bytes_hex": "203d", "token_id": 314, "utf8_fragment": " =" }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "202d", "token_id": 533, "utf8_fragment": " -" }, { "bytes_hex": "20636163686564", "token_id": 39472, "utf8_fragment": " cached" }, { "bytes_hex": "5f696e707574", "token_id": 12507, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "20706572", "token_id": 777, "utf8_fragment": " per" }, { "bytes_hex": "5f72657175657374", "token_id": 17371, "utf8_fragment": "_request" }, { "bytes_hex": "203d", "token_id": 314, "utf8_fragment": " =" }, { "bytes_hex": "20280a", "token_id": 4112, "utf8_fragment": " (\n" }, { "bytes_hex": "20202020202020", "token_id": 309, "utf8_fragment": " " }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "28", "token_id": 7, "utf8_fragment": "(" }, { "bytes_hex": "756e63", "token_id": 1922, "utf8_fragment": "unc" }, { "bytes_hex": "6163686564", "token_id": 5908, "utf8_fragment": "ached" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 425, "utf8_fragment": " *" }, { "bytes_hex": "20696e707574", "token_id": 3422, "utf8_fragment": " input" }, { "bytes_hex": "5f72617465", "token_id": 20477, "utf8_fragment": "_rate" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "20202020202020", "token_id": 309, "utf8_fragment": " " }, { "bytes_hex": "202b", "token_id": 659, "utf8_fragment": " +" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "2863", "token_id": 2410, "utf8_fragment": "(c" }, { "bytes_hex": "6163686564", "token_id": 5908, "utf8_fragment": "ached" }, { "bytes_hex": "5f696e707574", "token_id": 12507, "utf8_fragment": "_input" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 425, "utf8_fragment": " *" }, { "bytes_hex": "20636163686564", "token_id": 39472, "utf8_fragment": " cached" }, { "bytes_hex": "5f72617465", "token_id": 20477, "utf8_fragment": "_rate" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "20202020202020", "token_id": 309, "utf8_fragment": " " }, { "bytes_hex": "202b", "token_id": 659, "utf8_fragment": " +" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "286f7574707574", "token_id": 22485, "utf8_fragment": "(output" }, { "bytes_hex": "5f746f6b656e73", "token_id": 65830, "utf8_fragment": "_tokens" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 425, "utf8_fragment": " *" }, { "bytes_hex": "206f7574707574", "token_id": 4733, "utf8_fragment": " output" }, { "bytes_hex": "5f72617465", "token_id": 20477, "utf8_fragment": "_rate" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "2029", "token_id": 1546, "utf8_fragment": " )" }, { "bytes_hex": "202f", "token_id": 820, "utf8_fragment": " /" }, { "bytes_hex": "204d494c4c", "token_id": 129778, "utf8_fragment": " MILL" }, { "bytes_hex": "494f4e", "token_id": 2044, "utf8_fragment": "ION" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "202020", "token_id": 271, "utf8_fragment": " " }, { "bytes_hex": "2072657475726e", "token_id": 622, "utf8_fragment": " return" }, { "bytes_hex": "20446563696d616c", "token_id": 56874, "utf8_fragment": " Decimal" }, { "bytes_hex": "2872657175657374", "token_id": 9108, "utf8_fragment": "(request" }, { "bytes_hex": "73", "token_id": 82, "utf8_fragment": "s" }, { "bytes_hex": "29", "token_id": 8, "utf8_fragment": ")" }, { "bytes_hex": "202a", "token_id": 425, "utf8_fragment": " *" }, { "bytes_hex": "20706572", "token_id": 777, "utf8_fragment": " per" }, { "bytes_hex": "5f72657175657374", "token_id": 17371, "utf8_fragment": "_request" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" } ], "sample_id": "codigo", "source_sha256": "e576659845158bb30dc9d8443970e284418f7308c2df3ef4b91966aa14f043d0", "token_ids": [ 2845, 26964, 1588, 56874, 2499, 44, 16050, 2044, 314, 56874, 568, 1353, 1302, 15, 57903, 1314, 16213, 36820, 1607, 271, 13782, 25, 677, 412, 271, 3422, 65830, 25, 677, 412, 271, 39472, 12507, 65830, 25, 677, 412, 271, 4733, 65830, 25, 677, 412, 271, 3422, 20477, 25, 56874, 412, 271, 39472, 20477, 25, 56874, 412, 271, 4733, 20477, 25, 56874, 412, 8, 2747, 56874, 734, 271, 9280, 96726, 261, 16213, 6602, 8363, 591, 23141, 9837, 33044, 271, 538, 625, 220, 15, 5017, 39472, 12507, 65830, 5017, 3422, 65830, 734, 309, 9338, 9963, 2255, 568, 62023, 3422, 2804, 413, 261, 58845, 328, 3422, 10822, 271, 34591, 5908, 314, 3422, 65830, 533, 39472, 12507, 65830, 198, 271, 777, 17371, 314, 4112, 309, 56874, 7, 1922, 5908, 8, 425, 3422, 20477, 198, 309, 659, 56874, 2410, 5908, 12507, 65830, 8, 425, 39472, 20477, 198, 309, 659, 56874, 22485, 65830, 8, 425, 4733, 20477, 198, 271, 1546, 820, 129778, 2044, 198, 271, 622, 56874, 9108, 82, 8, 425, 777, 17371, 198 ] }, { "encoding": "cl100k_base", "pieces": [ { "bytes_hex": "41", "token_id": 32, "utf8_fragment": "A" }, { "bytes_hex": "206e65696768626f72686f6f64", "token_id": 12818, "utf8_fragment": " neighborhood" }, { "bytes_hex": "20636f6f7065726174697665", "token_id": 48566, "utf8_fragment": " cooperative" }, { "bytes_hex": "207072657061726573", "token_id": 48542, "utf8_fragment": " prepares" }, { "bytes_hex": "20697473", "token_id": 1202, "utf8_fragment": " its" }, { "bytes_hex": "20696e76656e746f7279", "token_id": 15808, "utf8_fragment": " inventory" }, { "bytes_hex": "206265666f7265", "token_id": 1603, "utf8_fragment": " before" }, { "bytes_hex": "206f70656e696e67", "token_id": 8736, "utf8_fragment": " opening" }, { "bytes_hex": "20697473", "token_id": 1202, "utf8_fragment": " its" }, { "bytes_hex": "206f6e6c696e65", "token_id": 2930, "utf8_fragment": " online" }, { "bytes_hex": "2073746f7265", "token_id": 3637, "utf8_fragment": " store" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20546865", "token_id": 578, "utf8_fragment": " The" }, { "bytes_hex": "207465616d", "token_id": 2128, "utf8_fragment": " team" }, { "bytes_hex": "2072657669657773", "token_id": 8544, "utf8_fragment": " reviews" }, { "bytes_hex": "206e616d6573", "token_id": 5144, "utf8_fragment": " names" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "206d6561737572656d656e7473", "token_id": 22323, "utf8_fragment": " measurements" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20616e64", "token_id": 323, "utf8_fragment": " and" }, { "bytes_hex": "20707269636573", "token_id": 7729, "utf8_fragment": " prices" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "2072656d6f766573", "token_id": 29260, "utf8_fragment": " removes" }, { "bytes_hex": "206475706c696361746573", "token_id": 43428, "utf8_fragment": " duplicates" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20616e64", "token_id": 323, "utf8_fragment": " and" }, { "bytes_hex": "207265636f726473", "token_id": 7576, "utf8_fragment": " records" }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "2064617465", "token_id": 2457, "utf8_fragment": " date" }, { "bytes_hex": "206f66", "token_id": 315, "utf8_fragment": " of" }, { "bytes_hex": "206576657279", "token_id": 1475, "utf8_fragment": " every" }, { "bytes_hex": "206368616e6765", "token_id": 2349, "utf8_fragment": " change" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "204974", "token_id": 1102, "utf8_fragment": " It" }, { "bytes_hex": "207468656e", "token_id": 1243, "utf8_fragment": " then" }, { "bytes_hex": "20636f6d7061726573", "token_id": 43565, "utf8_fragment": " compares" }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "207265706f7274", "token_id": 1934, "utf8_fragment": " report" }, { "bytes_hex": "2077697468", "token_id": 449, "utf8_fragment": " with" }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "206f726967696e616c", "token_id": 4113, "utf8_fragment": " original" }, { "bytes_hex": "20696e766f69636573", "token_id": 65152, "utf8_fragment": " invoices" }, { "bytes_hex": "20616e64", "token_id": 323, "utf8_fragment": " and" }, { "bytes_hex": "207265736572766573", "token_id": 30600, "utf8_fragment": " reserves" }, { "bytes_hex": "206f6e65", "token_id": 832, "utf8_fragment": " one" }, { "bytes_hex": "20636f6c756d6e", "token_id": 3330, "utf8_fragment": " column" }, { "bytes_hex": "20666f72", "token_id": 369, "utf8_fragment": " for" }, { "bytes_hex": "206f62736572766174696f6e73", "token_id": 24654, "utf8_fragment": " observations" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20546865", "token_id": 578, "utf8_fragment": " The" }, { "bytes_hex": "20676f616c", "token_id": 5915, "utf8_fragment": " goal" }, { "bytes_hex": "206973", "token_id": 374, "utf8_fragment": " is" }, { "bytes_hex": "206e6f74", "token_id": 539, "utf8_fragment": " not" }, { "bytes_hex": "20746f", "token_id": 311, "utf8_fragment": " to" }, { "bytes_hex": "2066696c6c", "token_id": 5266, "utf8_fragment": " fill" }, { "bytes_hex": "2061", "token_id": 264, "utf8_fragment": " a" }, { "bytes_hex": "2068756765", "token_id": 6908, "utf8_fragment": " huge" }, { "bytes_hex": "207461626c65", "token_id": 2007, "utf8_fragment": " table" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20627574", "token_id": 719, "utf8_fragment": " but" }, { "bytes_hex": "20746f", "token_id": 311, "utf8_fragment": " to" }, { "bytes_hex": "207072657365727665", "token_id": 21813, "utf8_fragment": " preserve" }, { "bytes_hex": "20656e6f756768", "token_id": 3403, "utf8_fragment": " enough" }, { "bytes_hex": "2064617461", "token_id": 828, "utf8_fragment": " data" }, { "bytes_hex": "20746f", "token_id": 311, "utf8_fragment": " to" }, { "bytes_hex": "20616e73776572", "token_id": 4320, "utf8_fragment": " answer" }, { "bytes_hex": "207370656369666963", "token_id": 3230, "utf8_fragment": " specific" }, { "bytes_hex": "207175657374696f6e73", "token_id": 4860, "utf8_fragment": " questions" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "204966", "token_id": 1442, "utf8_fragment": " If" }, { "bytes_hex": "2061", "token_id": 264, "utf8_fragment": " a" }, { "bytes_hex": "2064697363726570616e6379", "token_id": 79105, "utf8_fragment": " discrepancy" }, { "bytes_hex": "2061707065617273", "token_id": 8111, "utf8_fragment": " appears" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "20726573706f6e7369626c65", "token_id": 8647, "utf8_fragment": " responsible" }, { "bytes_hex": "20706572736f6e", "token_id": 1732, "utf8_fragment": " person" }, { "bytes_hex": "2072657475726e73", "token_id": 4780, "utf8_fragment": " returns" }, { "bytes_hex": "20746f", "token_id": 311, "utf8_fragment": " to" }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "20736f75726365", "token_id": 2592, "utf8_fragment": " source" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20636f7272656374", "token_id": 4495, "utf8_fragment": " correct" }, { "bytes_hex": "73", "token_id": 82, "utf8_fragment": "s" }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "207265636f7264", "token_id": 3335, "utf8_fragment": " record" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20616e64", "token_id": 323, "utf8_fragment": " and" }, { "bytes_hex": "206c6561766573", "token_id": 11141, "utf8_fragment": " leaves" }, { "bytes_hex": "2061", "token_id": 264, "utf8_fragment": " a" }, { "bytes_hex": "2073686f7274", "token_id": 2875, "utf8_fragment": " short" }, { "bytes_hex": "206e6f7465", "token_id": 5296, "utf8_fragment": " note" }, { "bytes_hex": "20666f72", "token_id": 369, "utf8_fragment": " for" }, { "bytes_hex": "20746865", "token_id": 279, "utf8_fragment": " the" }, { "bytes_hex": "206e657874", "token_id": 1828, "utf8_fragment": " next" }, { "bytes_hex": "20726576696577", "token_id": 3477, "utf8_fragment": " review" }, { "bytes_hex": "2e0a", "token_id": 627, "utf8_fragment": ".\n" } ], "sample_id": "en-prose", "source_sha256": "47034166749994459dccd3e308934eb9d9bdff965d4bfc2f03518b3cea92ce29", "token_ids": [ 32, 12818, 48566, 48542, 1202, 15808, 1603, 8736, 1202, 2930, 3637, 13, 578, 2128, 8544, 5144, 11, 22323, 11, 323, 7729, 11, 29260, 43428, 11, 323, 7576, 279, 2457, 315, 1475, 2349, 13, 1102, 1243, 43565, 279, 1934, 449, 279, 4113, 65152, 323, 30600, 832, 3330, 369, 24654, 13, 578, 5915, 374, 539, 311, 5266, 264, 6908, 2007, 11, 719, 311, 21813, 3403, 828, 311, 4320, 3230, 4860, 13, 1442, 264, 79105, 8111, 11, 279, 8647, 1732, 4780, 311, 279, 2592, 11, 4495, 82, 279, 3335, 11, 323, 11141, 264, 2875, 5296, 369, 279, 1828, 3477, 627 ] }, { "encoding": "o200k_base", "pieces": [ { "bytes_hex": "41", "token_id": 32, "utf8_fragment": "A" }, { "bytes_hex": "206e65696768626f72686f6f64", "token_id": 17528, "utf8_fragment": " neighborhood" }, { "bytes_hex": "20636f6f7065726174697665", "token_id": 76489, "utf8_fragment": " cooperative" }, { "bytes_hex": "207072657061726573", "token_id": 77064, "utf8_fragment": " prepares" }, { "bytes_hex": "20697473", "token_id": 1617, "utf8_fragment": " its" }, { "bytes_hex": "20696e76656e746f7279", "token_id": 21752, "utf8_fragment": " inventory" }, { "bytes_hex": "206265666f7265", "token_id": 2254, "utf8_fragment": " before" }, { "bytes_hex": "206f70656e696e67", "token_id": 12218, "utf8_fragment": " opening" }, { "bytes_hex": "20697473", "token_id": 1617, "utf8_fragment": " its" }, { "bytes_hex": "206f6e6c696e65", "token_id": 2296, "utf8_fragment": " online" }, { "bytes_hex": "2073746f7265", "token_id": 4897, "utf8_fragment": " store" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20546865", "token_id": 623, "utf8_fragment": " The" }, { "bytes_hex": "207465616d", "token_id": 3214, "utf8_fragment": " team" }, { "bytes_hex": "2072657669657773", "token_id": 7042, "utf8_fragment": " reviews" }, { "bytes_hex": "206e616d6573", "token_id": 8747, "utf8_fragment": " names" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "206d6561737572656d656e7473", "token_id": 29039, "utf8_fragment": " measurements" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20616e64", "token_id": 326, "utf8_fragment": " and" }, { "bytes_hex": "20707269636573", "token_id": 8451, "utf8_fragment": " prices" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "2072656d6f766573", "token_id": 52683, "utf8_fragment": " removes" }, { "bytes_hex": "206475706c696361746573", "token_id": 91135, "utf8_fragment": " duplicates" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20616e64", "token_id": 326, "utf8_fragment": " and" }, { "bytes_hex": "207265636f726473", "token_id": 11722, "utf8_fragment": " records" }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "2064617465", "token_id": 3521, "utf8_fragment": " date" }, { "bytes_hex": "206f66", "token_id": 328, "utf8_fragment": " of" }, { "bytes_hex": "206576657279", "token_id": 1753, "utf8_fragment": " every" }, { "bytes_hex": "206368616e6765", "token_id": 3343, "utf8_fragment": " change" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "204974", "token_id": 1225, "utf8_fragment": " It" }, { "bytes_hex": "207468656e", "token_id": 1815, "utf8_fragment": " then" }, { "bytes_hex": "20636f6d7061726573", "token_id": 70814, "utf8_fragment": " compares" }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "207265706f7274", "token_id": 3019, "utf8_fragment": " report" }, { "bytes_hex": "2077697468", "token_id": 483, "utf8_fragment": " with" }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "206f726967696e616c", "token_id": 4756, "utf8_fragment": " original" }, { "bytes_hex": "20696e766f69636573", "token_id": 87586, "utf8_fragment": " invoices" }, { "bytes_hex": "20616e64", "token_id": 326, "utf8_fragment": " and" }, { "bytes_hex": "207265736572766573", "token_id": 41452, "utf8_fragment": " reserves" }, { "bytes_hex": "206f6e65", "token_id": 1001, "utf8_fragment": " one" }, { "bytes_hex": "20636f6c756d6e", "token_id": 6000, "utf8_fragment": " column" }, { "bytes_hex": "20666f72", "token_id": 395, "utf8_fragment": " for" }, { "bytes_hex": "206f62736572766174696f6e73", "token_id": 39105, "utf8_fragment": " observations" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20546865", "token_id": 623, "utf8_fragment": " The" }, { "bytes_hex": "20676f616c", "token_id": 8583, "utf8_fragment": " goal" }, { "bytes_hex": "206973", "token_id": 382, "utf8_fragment": " is" }, { "bytes_hex": "206e6f74", "token_id": 625, "utf8_fragment": " not" }, { "bytes_hex": "20746f", "token_id": 316, "utf8_fragment": " to" }, { "bytes_hex": "2066696c6c", "token_id": 6954, "utf8_fragment": " fill" }, { "bytes_hex": "2061", "token_id": 261, "utf8_fragment": " a" }, { "bytes_hex": "2068756765", "token_id": 9560, "utf8_fragment": " huge" }, { "bytes_hex": "207461626c65", "token_id": 3293, "utf8_fragment": " table" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20627574", "token_id": 889, "utf8_fragment": " but" }, { "bytes_hex": "20746f", "token_id": 316, "utf8_fragment": " to" }, { "bytes_hex": "207072657365727665", "token_id": 33507, "utf8_fragment": " preserve" }, { "bytes_hex": "20656e6f756768", "token_id": 4951, "utf8_fragment": " enough" }, { "bytes_hex": "2064617461", "token_id": 1238, "utf8_fragment": " data" }, { "bytes_hex": "20746f", "token_id": 316, "utf8_fragment": " to" }, { "bytes_hex": "20616e73776572", "token_id": 6052, "utf8_fragment": " answer" }, { "bytes_hex": "207370656369666963", "token_id": 4857, "utf8_fragment": " specific" }, { "bytes_hex": "207175657374696f6e73", "token_id": 5359, "utf8_fragment": " questions" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "204966", "token_id": 1843, "utf8_fragment": " If" }, { "bytes_hex": "2061", "token_id": 261, "utf8_fragment": " a" }, { "bytes_hex": "2064697363726570616e6379", "token_id": 173834, "utf8_fragment": " discrepancy" }, { "bytes_hex": "2061707065617273", "token_id": 14518, "utf8_fragment": " appears" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "20726573706f6e7369626c65", "token_id": 10908, "utf8_fragment": " responsible" }, { "bytes_hex": "20706572736f6e", "token_id": 1647, "utf8_fragment": " person" }, { "bytes_hex": "2072657475726e73", "token_id": 7377, "utf8_fragment": " returns" }, { "bytes_hex": "20746f", "token_id": 316, "utf8_fragment": " to" }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "20736f75726365", "token_id": 4256, "utf8_fragment": " source" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20636f7272656374", "token_id": 6145, "utf8_fragment": " correct" }, { "bytes_hex": "73", "token_id": 82, "utf8_fragment": "s" }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "207265636f7264", "token_id": 3704, "utf8_fragment": " record" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20616e64", "token_id": 326, "utf8_fragment": " and" }, { "bytes_hex": "206c6561766573", "token_id": 15657, "utf8_fragment": " leaves" }, { "bytes_hex": "2061", "token_id": 261, "utf8_fragment": " a" }, { "bytes_hex": "2073686f7274", "token_id": 4022, "utf8_fragment": " short" }, { "bytes_hex": "206e6f7465", "token_id": 7477, "utf8_fragment": " note" }, { "bytes_hex": "20666f72", "token_id": 395, "utf8_fragment": " for" }, { "bytes_hex": "20746865", "token_id": 290, "utf8_fragment": " the" }, { "bytes_hex": "206e657874", "token_id": 2613, "utf8_fragment": " next" }, { "bytes_hex": "20726576696577", "token_id": 3358, "utf8_fragment": " review" }, { "bytes_hex": "2e0a", "token_id": 558, "utf8_fragment": ".\n" } ], "sample_id": "en-prose", "source_sha256": "47034166749994459dccd3e308934eb9d9bdff965d4bfc2f03518b3cea92ce29", "token_ids": [ 32, 17528, 76489, 77064, 1617, 21752, 2254, 12218, 1617, 2296, 4897, 13, 623, 3214, 7042, 8747, 11, 29039, 11, 326, 8451, 11, 52683, 91135, 11, 326, 11722, 290, 3521, 328, 1753, 3343, 13, 1225, 1815, 70814, 290, 3019, 483, 290, 4756, 87586, 326, 41452, 1001, 6000, 395, 39105, 13, 623, 8583, 382, 625, 316, 6954, 261, 9560, 3293, 11, 889, 316, 33507, 4951, 1238, 316, 6052, 4857, 5359, 13, 1843, 261, 173834, 14518, 11, 290, 10908, 1647, 7377, 316, 290, 4256, 11, 6145, 82, 290, 3704, 11, 326, 15657, 261, 4022, 7477, 395, 290, 2613, 3358, 558 ] }, { "encoding": "cl100k_base", "pieces": [ { "bytes_hex": "556e61", "token_id": 65194, "utf8_fragment": "Una" }, { "bytes_hex": "20636f6f706572", "token_id": 22415, "utf8_fragment": " cooper" }, { "bytes_hex": "6174697661", "token_id": 28952, "utf8_fragment": "ativa" }, { "bytes_hex": "206465", "token_id": 409, "utf8_fragment": " de" }, { "bytes_hex": "20626172", "token_id": 3703, "utf8_fragment": " bar" }, { "bytes_hex": "72696f", "token_id": 10599, "utf8_fragment": "rio" }, { "bytes_hex": "20707265706172", "token_id": 10073, "utf8_fragment": " prepar" }, { "bytes_hex": "61", "token_id": 64, "utf8_fragment": "a" }, { "bytes_hex": "20656c", "token_id": 658, "utf8_fragment": " el" }, { "bytes_hex": "20696e76656e74", "token_id": 17459, "utf8_fragment": " invent" }, { "bytes_hex": "6172696f", "token_id": 3370, "utf8_fragment": "ario" }, { "bytes_hex": "20616e746573", "token_id": 34435, "utf8_fragment": " antes" }, { "bytes_hex": "206465", "token_id": 409, "utf8_fragment": " de" }, { "bytes_hex": "206162726972", "token_id": 73663, "utf8_fragment": " abrir" }, { "bytes_hex": "207375", "token_id": 924, "utf8_fragment": " su" }, { "bytes_hex": "207469", "token_id": 9165, "utf8_fragment": " ti" }, { "bytes_hex": "656e6461", "token_id": 9895, "utf8_fragment": "enda" }, { "bytes_hex": "20656e", "token_id": 665, "utf8_fragment": " en" }, { "bytes_hex": "206cc3ad6e6561", "token_id": 56262, "utf8_fragment": " línea" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20456c", "token_id": 4072, "utf8_fragment": " El" }, { "bytes_hex": "2065717569706f", "token_id": 49012, "utf8_fragment": " equipo" }, { "bytes_hex": "207265766973", "token_id": 17951, "utf8_fragment": " revis" }, { "bytes_hex": "61", "token_id": 64, "utf8_fragment": "a" }, { "bytes_hex": "206e6f6d62726573", "token_id": 81698, "utf8_fragment": " nombres" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "206d656469646173", "token_id": 83120, "utf8_fragment": " medidas" }, { "bytes_hex": "2079", "token_id": 379, "utf8_fragment": " y" }, { "bytes_hex": "2070726563696f73", "token_id": 84881, "utf8_fragment": " precios" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20656c696d", "token_id": 9961, "utf8_fragment": " elim" }, { "bytes_hex": "696e61", "token_id": 2259, "utf8_fragment": "ina" }, { "bytes_hex": "206475706c6963", "token_id": 27444, "utf8_fragment": " duplic" }, { "bytes_hex": "61646f73", "token_id": 5670, "utf8_fragment": "ados" }, { "bytes_hex": "2079", "token_id": 379, "utf8_fragment": " y" }, { "bytes_hex": "20616e", "token_id": 459, "utf8_fragment": " an" }, { "bytes_hex": "6f7461", "token_id": 6217, "utf8_fragment": "ota" }, { "bytes_hex": "206c61", "token_id": 1208, "utf8_fragment": " la" }, { "bytes_hex": "206665636861", "token_id": 23894, "utf8_fragment": " fecha" }, { "bytes_hex": "206465", "token_id": 409, "utf8_fragment": " de" }, { "bytes_hex": "2063616461", "token_id": 19394, "utf8_fragment": " cada" }, { "bytes_hex": "2063616d62696f", "token_id": 58411, "utf8_fragment": " cambio" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20446573", "token_id": 3959, "utf8_fragment": " Des" }, { "bytes_hex": "7075c3a973", "token_id": 43309, "utf8_fragment": "pués" }, { "bytes_hex": "20636f6d70", "token_id": 1391, "utf8_fragment": " comp" }, { "bytes_hex": "617261", "token_id": 5169, "utf8_fragment": "ara" }, { "bytes_hex": "20656c", "token_id": 658, "utf8_fragment": " el" }, { "bytes_hex": "20696e666f726d", "token_id": 6179, "utf8_fragment": " inform" }, { "bytes_hex": "65", "token_id": 68, "utf8_fragment": "e" }, { "bytes_hex": "20636f6e", "token_id": 390, "utf8_fragment": " con" }, { "bytes_hex": "206c6173", "token_id": 5252, "utf8_fragment": " las" }, { "bytes_hex": "2066616374", "token_id": 2144, "utf8_fragment": " fact" }, { "bytes_hex": "75726173", "token_id": 19879, "utf8_fragment": "uras" }, { "bytes_hex": "206f726967", "token_id": 2780, "utf8_fragment": " orig" }, { "bytes_hex": "696e61", "token_id": 2259, "utf8_fragment": "ina" }, { "bytes_hex": "6c6573", "token_id": 645, "utf8_fragment": "les" }, { "bytes_hex": "2079", "token_id": 379, "utf8_fragment": " y" }, { "bytes_hex": "2072657365727661", "token_id": 99744, "utf8_fragment": " reserva" }, { "bytes_hex": "20756e61", "token_id": 5203, "utf8_fragment": " una" }, { "bytes_hex": "20636f6c756d6e61", "token_id": 76399, "utf8_fragment": " columna" }, { "bytes_hex": "2070617261", "token_id": 3429, "utf8_fragment": " para" }, { "bytes_hex": "206f6273657276", "token_id": 9466, "utf8_fragment": " observ" }, { "bytes_hex": "6163696f6e6573", "token_id": 12712, "utf8_fragment": "aciones" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20456c", "token_id": 4072, "utf8_fragment": " El" }, { "bytes_hex": "206f626a657469766f", "token_id": 55326, "utf8_fragment": " objetivo" }, { "bytes_hex": "206e6f", "token_id": 912, "utf8_fragment": " no" }, { "bytes_hex": "206573", "token_id": 1560, "utf8_fragment": " es" }, { "bytes_hex": "206c6c656e", "token_id": 79931, "utf8_fragment": " llen" }, { "bytes_hex": "6172", "token_id": 277, "utf8_fragment": "ar" }, { "bytes_hex": "20756e61", "token_id": 5203, "utf8_fragment": " una" }, { "bytes_hex": "207461626c61", "token_id": 41646, "utf8_fragment": " tabla" }, { "bytes_hex": "20656e6f726d65", "token_id": 68906, "utf8_fragment": " enorme" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "2073696e6f", "token_id": 52904, "utf8_fragment": " sino" }, { "bytes_hex": "20636f6e73657276", "token_id": 11357, "utf8_fragment": " conserv" }, { "bytes_hex": "6172", "token_id": 277, "utf8_fragment": "ar" }, { "bytes_hex": "206461746f73", "token_id": 18224, "utf8_fragment": " datos" }, { "bytes_hex": "20737566", "token_id": 56868, "utf8_fragment": " suf" }, { "bytes_hex": "696369656e74", "token_id": 5499, "utf8_fragment": "icient" }, { "bytes_hex": "6573", "token_id": 288, "utf8_fragment": "es" }, { "bytes_hex": "2070617261", "token_id": 3429, "utf8_fragment": " para" }, { "bytes_hex": "20726573706f6e646572", "token_id": 65134, "utf8_fragment": " responder" }, { "bytes_hex": "2070726567", "token_id": 20575, "utf8_fragment": " preg" }, { "bytes_hex": "756e746173", "token_id": 55672, "utf8_fragment": "untas" }, { "bytes_hex": "20636f6e", "token_id": 390, "utf8_fragment": " con" }, { "bytes_hex": "63726574", "token_id": 4523, "utf8_fragment": "cret" }, { "bytes_hex": "6173", "token_id": 300, "utf8_fragment": "as" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "205369", "token_id": 12095, "utf8_fragment": " Si" }, { "bytes_hex": "206170617265", "token_id": 56270, "utf8_fragment": " apare" }, { "bytes_hex": "6365", "token_id": 346, "utf8_fragment": "ce" }, { "bytes_hex": "20756e61", "token_id": 5203, "utf8_fragment": " una" }, { "bytes_hex": "206469666572656e636961", "token_id": 95527, "utf8_fragment": " diferencia" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "206c61", "token_id": 1208, "utf8_fragment": " la" }, { "bytes_hex": "20706572736f6e61", "token_id": 29055, "utf8_fragment": " persona" }, { "bytes_hex": "20726573706f6e7361626c65", "token_id": 77932, "utf8_fragment": " responsable" }, { "bytes_hex": "2076", "token_id": 348, "utf8_fragment": " v" }, { "bytes_hex": "75656c7665", "token_id": 47956, "utf8_fragment": "uelve" }, { "bytes_hex": "2061", "token_id": 264, "utf8_fragment": " a" }, { "bytes_hex": "206c61", "token_id": 1208, "utf8_fragment": " la" }, { "bytes_hex": "206675", "token_id": 18922, "utf8_fragment": " fu" }, { "bytes_hex": "656e7465", "token_id": 6960, "utf8_fragment": "ente" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20636f7272", "token_id": 45453, "utf8_fragment": " corr" }, { "bytes_hex": "696765", "token_id": 7404, "utf8_fragment": "ige" }, { "bytes_hex": "20656c", "token_id": 658, "utf8_fragment": " el" }, { "bytes_hex": "20726567697374726f", "token_id": 37885, "utf8_fragment": " registro" }, { "bytes_hex": "2079", "token_id": 379, "utf8_fragment": " y" }, { "bytes_hex": "2064656a61", "token_id": 76733, "utf8_fragment": " deja" }, { "bytes_hex": "20756e61", "token_id": 5203, "utf8_fragment": " una" }, { "bytes_hex": "206e6f7461", "token_id": 45138, "utf8_fragment": " nota" }, { "bytes_hex": "206272657665", "token_id": 98519, "utf8_fragment": " breve" }, { "bytes_hex": "2070617261", "token_id": 3429, "utf8_fragment": " para" }, { "bytes_hex": "206c61", "token_id": 1208, "utf8_fragment": " la" }, { "bytes_hex": "207072c3b378696d61", "token_id": 91307, "utf8_fragment": " próxima" }, { "bytes_hex": "207265766973", "token_id": 17951, "utf8_fragment": " revis" }, { "bytes_hex": "69c3b36e", "token_id": 3739, "utf8_fragment": "ión" }, { "bytes_hex": "2e0a", "token_id": 627, "utf8_fragment": ".\n" } ], "sample_id": "es-prosa", "source_sha256": "d3feea1cbcfc37da0ad6c510052d679610a1dd48c5d25a3bd718c8c2733a8ff3", "token_ids": [ 65194, 22415, 28952, 409, 3703, 10599, 10073, 64, 658, 17459, 3370, 34435, 409, 73663, 924, 9165, 9895, 665, 56262, 13, 4072, 49012, 17951, 64, 81698, 11, 83120, 379, 84881, 11, 9961, 2259, 27444, 5670, 379, 459, 6217, 1208, 23894, 409, 19394, 58411, 13, 3959, 43309, 1391, 5169, 658, 6179, 68, 390, 5252, 2144, 19879, 2780, 2259, 645, 379, 99744, 5203, 76399, 3429, 9466, 12712, 13, 4072, 55326, 912, 1560, 79931, 277, 5203, 41646, 68906, 11, 52904, 11357, 277, 18224, 56868, 5499, 288, 3429, 65134, 20575, 55672, 390, 4523, 300, 13, 12095, 56270, 346, 5203, 95527, 11, 1208, 29055, 77932, 348, 47956, 264, 1208, 18922, 6960, 11, 45453, 7404, 658, 37885, 379, 76733, 5203, 45138, 98519, 3429, 1208, 91307, 17951, 3739, 627 ] }, { "encoding": "o200k_base", "pieces": [ { "bytes_hex": "556e61", "token_id": 26136, "utf8_fragment": "Una" }, { "bytes_hex": "20636f6f706572", "token_id": 28520, "utf8_fragment": " cooper" }, { "bytes_hex": "6174697661", "token_id": 10277, "utf8_fragment": "ativa" }, { "bytes_hex": "206465", "token_id": 334, "utf8_fragment": " de" }, { "bytes_hex": "2062617272696f", "token_id": 82960, "utf8_fragment": " barrio" }, { "bytes_hex": "2070726570617261", "token_id": 114102, "utf8_fragment": " prepara" }, { "bytes_hex": "20656c", "token_id": 650, "utf8_fragment": " el" }, { "bytes_hex": "20696e76656e74", "token_id": 20420, "utf8_fragment": " invent" }, { "bytes_hex": "6172696f", "token_id": 3544, "utf8_fragment": "ario" }, { "bytes_hex": "20616e746573", "token_id": 13290, "utf8_fragment": " antes" }, { "bytes_hex": "206465", "token_id": 334, "utf8_fragment": " de" }, { "bytes_hex": "206162726972", "token_id": 41731, "utf8_fragment": " abrir" }, { "bytes_hex": "207375", "token_id": 593, "utf8_fragment": " su" }, { "bytes_hex": "207469656e6461", "token_id": 60928, "utf8_fragment": " tienda" }, { "bytes_hex": "20656e", "token_id": 469, "utf8_fragment": " en" }, { "bytes_hex": "206cc3ad6e6561", "token_id": 31293, "utf8_fragment": " línea" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20456c", "token_id": 3241, "utf8_fragment": " El" }, { "bytes_hex": "2065717569706f", "token_id": 22118, "utf8_fragment": " equipo" }, { "bytes_hex": "207265766973", "token_id": 19497, "utf8_fragment": " revis" }, { "bytes_hex": "61", "token_id": 64, "utf8_fragment": "a" }, { "bytes_hex": "206e6f6d62726573", "token_id": 76205, "utf8_fragment": " nombres" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "206d656469646173", "token_id": 34009, "utf8_fragment": " medidas" }, { "bytes_hex": "2079", "token_id": 342, "utf8_fragment": " y" }, { "bytes_hex": "2070726563696f73", "token_id": 40409, "utf8_fragment": " precios" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20656c696d696e61", "token_id": 144540, "utf8_fragment": " elimina" }, { "bytes_hex": "206475706c6963", "token_id": 25761, "utf8_fragment": " duplic" }, { "bytes_hex": "61646f73", "token_id": 2426, "utf8_fragment": "ados" }, { "bytes_hex": "2079", "token_id": 342, "utf8_fragment": " y" }, { "bytes_hex": "20616e", "token_id": 448, "utf8_fragment": " an" }, { "bytes_hex": "6f7461", "token_id": 5852, "utf8_fragment": "ota" }, { "bytes_hex": "206c61", "token_id": 557, "utf8_fragment": " la" }, { "bytes_hex": "206665636861", "token_id": 26375, "utf8_fragment": " fecha" }, { "bytes_hex": "206465", "token_id": 334, "utf8_fragment": " de" }, { "bytes_hex": "2063616461", "token_id": 8272, "utf8_fragment": " cada" }, { "bytes_hex": "2063616d62696f", "token_id": 30643, "utf8_fragment": " cambio" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "204465737075c3a973", "token_id": 79408, "utf8_fragment": " Después" }, { "bytes_hex": "20636f6d70617261", "token_id": 174599, "utf8_fragment": " compara" }, { "bytes_hex": "20656c", "token_id": 650, "utf8_fragment": " el" }, { "bytes_hex": "20696e666f726d65", "token_id": 55247, "utf8_fragment": " informe" }, { "bytes_hex": "20636f6e", "token_id": 406, "utf8_fragment": " con" }, { "bytes_hex": "206c6173", "token_id": 1996, "utf8_fragment": " las" }, { "bytes_hex": "2066616374", "token_id": 2840, "utf8_fragment": " fact" }, { "bytes_hex": "75726173", "token_id": 12558, "utf8_fragment": "uras" }, { "bytes_hex": "206f726967696e616c6573", "token_id": 107158, "utf8_fragment": " originales" }, { "bytes_hex": "2079", "token_id": 342, "utf8_fragment": " y" }, { "bytes_hex": "2072657365727661", "token_id": 65127, "utf8_fragment": " reserva" }, { "bytes_hex": "20756e61", "token_id": 1969, "utf8_fragment": " una" }, { "bytes_hex": "20636f6c756d6e61", "token_id": 104382, "utf8_fragment": " columna" }, { "bytes_hex": "2070617261", "token_id": 1209, "utf8_fragment": " para" }, { "bytes_hex": "206f6273657276", "token_id": 8664, "utf8_fragment": " observ" }, { "bytes_hex": "6163696f6e6573", "token_id": 5906, "utf8_fragment": "aciones" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "20456c", "token_id": 3241, "utf8_fragment": " El" }, { "bytes_hex": "206f626a657469766f", "token_id": 22309, "utf8_fragment": " objetivo" }, { "bytes_hex": "206e6f", "token_id": 860, "utf8_fragment": " no" }, { "bytes_hex": "206573", "token_id": 878, "utf8_fragment": " es" }, { "bytes_hex": "206c6c656e6172", "token_id": 173325, "utf8_fragment": " llenar" }, { "bytes_hex": "20756e61", "token_id": 1969, "utf8_fragment": " una" }, { "bytes_hex": "207461626c61", "token_id": 58018, "utf8_fragment": " tabla" }, { "bytes_hex": "20656e6f726d65", "token_id": 37949, "utf8_fragment": " enorme" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "2073696e6f", "token_id": 22808, "utf8_fragment": " sino" }, { "bytes_hex": "20636f6e736572766172", "token_id": 149057, "utf8_fragment": " conservar" }, { "bytes_hex": "206461746f73", "token_id": 18336, "utf8_fragment": " datos" }, { "bytes_hex": "20737566696369656e746573", "token_id": 123901, "utf8_fragment": " suficientes" }, { "bytes_hex": "2070617261", "token_id": 1209, "utf8_fragment": " para" }, { "bytes_hex": "20726573706f6e646572", "token_id": 51279, "utf8_fragment": " responder" }, { "bytes_hex": "2070726567756e746173", "token_id": 68980, "utf8_fragment": " preguntas" }, { "bytes_hex": "20636f6e63726574", "token_id": 54394, "utf8_fragment": " concret" }, { "bytes_hex": "6173", "token_id": 288, "utf8_fragment": "as" }, { "bytes_hex": "2e", "token_id": 13, "utf8_fragment": "." }, { "bytes_hex": "205369", "token_id": 7832, "utf8_fragment": " Si" }, { "bytes_hex": "2061706172656365", "token_id": 56869, "utf8_fragment": " aparece" }, { "bytes_hex": "20756e61", "token_id": 1969, "utf8_fragment": " una" }, { "bytes_hex": "206469666572656e636961", "token_id": 50158, "utf8_fragment": " diferencia" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "206c61", "token_id": 557, "utf8_fragment": " la" }, { "bytes_hex": "20706572736f6e61", "token_id": 19672, "utf8_fragment": " persona" }, { "bytes_hex": "20726573706f6e7361626c65", "token_id": 39439, "utf8_fragment": " responsable" }, { "bytes_hex": "207675656c7665", "token_id": 86553, "utf8_fragment": " vuelve" }, { "bytes_hex": "2061", "token_id": 261, "utf8_fragment": " a" }, { "bytes_hex": "206c61", "token_id": 557, "utf8_fragment": " la" }, { "bytes_hex": "206675656e7465", "token_id": 71680, "utf8_fragment": " fuente" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "20636f7272", "token_id": 16963, "utf8_fragment": " corr" }, { "bytes_hex": "696765", "token_id": 2609, "utf8_fragment": "ige" }, { "bytes_hex": "20656c", "token_id": 650, "utf8_fragment": " el" }, { "bytes_hex": "20726567697374726f", "token_id": 39764, "utf8_fragment": " registro" }, { "bytes_hex": "2079", "token_id": 342, "utf8_fragment": " y" }, { "bytes_hex": "2064656a61", "token_id": 47096, "utf8_fragment": " deja" }, { "bytes_hex": "20756e61", "token_id": 1969, "utf8_fragment": " una" }, { "bytes_hex": "206e6f7461", "token_id": 33016, "utf8_fragment": " nota" }, { "bytes_hex": "206272657665", "token_id": 55567, "utf8_fragment": " breve" }, { "bytes_hex": "2070617261", "token_id": 1209, "utf8_fragment": " para" }, { "bytes_hex": "206c61", "token_id": 557, "utf8_fragment": " la" }, { "bytes_hex": "207072c3b378696d61", "token_id": 46257, "utf8_fragment": " próxima" }, { "bytes_hex": "20726576697369c3b36e", "token_id": 124007, "utf8_fragment": " revisión" }, { "bytes_hex": "2e0a", "token_id": 558, "utf8_fragment": ".\n" } ], "sample_id": "es-prosa", "source_sha256": "d3feea1cbcfc37da0ad6c510052d679610a1dd48c5d25a3bd718c8c2733a8ff3", "token_ids": [ 26136, 28520, 10277, 334, 82960, 114102, 650, 20420, 3544, 13290, 334, 41731, 593, 60928, 469, 31293, 13, 3241, 22118, 19497, 64, 76205, 11, 34009, 342, 40409, 11, 144540, 25761, 2426, 342, 448, 5852, 557, 26375, 334, 8272, 30643, 13, 79408, 174599, 650, 55247, 406, 1996, 2840, 12558, 107158, 342, 65127, 1969, 104382, 1209, 8664, 5906, 13, 3241, 22309, 860, 878, 173325, 1969, 58018, 37949, 11, 22808, 149057, 18336, 123901, 1209, 51279, 68980, 54394, 288, 13, 7832, 56869, 1969, 50158, 11, 557, 19672, 39439, 86553, 261, 557, 71680, 11, 16963, 2609, 650, 39764, 342, 47096, 1969, 33016, 55567, 1209, 557, 46257, 124007, 558 ] }, { "encoding": "cl100k_base", "pieces": [ { "bytes_hex": "657363", "token_id": 3380, "utf8_fragment": "esc" }, { "bytes_hex": "656e6172696f", "token_id": 21044, "utf8_fragment": "enario" }, { "bytes_hex": "2c73", "token_id": 10900, "utf8_fragment": ",s" }, { "bytes_hex": "6f6c6963", "token_id": 7918, "utf8_fragment": "olic" }, { "bytes_hex": "697475646573", "token_id": 21237, "utf8_fragment": "itudes" }, { "bytes_hex": "5f6d6573", "token_id": 70259, "utf8_fragment": "_mes" }, { "bytes_hex": "2c74", "token_id": 14287, "utf8_fragment": ",t" }, { "bytes_hex": "6f6b656e73", "token_id": 9912, "utf8_fragment": "okens" }, { "bytes_hex": "5f", "token_id": 62, "utf8_fragment": "_" }, { "bytes_hex": "656e7472616461", "token_id": 63340, "utf8_fragment": "entrada" }, { "bytes_hex": "2c74", "token_id": 14287, "utf8_fragment": ",t" }, { "bytes_hex": "6f6b656e73", "token_id": 9912, "utf8_fragment": "okens" }, { "bytes_hex": "5f6361636865", "token_id": 11790, "utf8_fragment": "_cache" }, { "bytes_hex": "2c74", "token_id": 14287, "utf8_fragment": ",t" }, { "bytes_hex": "6f6b656e73", "token_id": 9912, "utf8_fragment": "okens" }, { "bytes_hex": "5f73616c", "token_id": 63591, "utf8_fragment": "_sal" }, { "bytes_hex": "696461", "token_id": 4849, "utf8_fragment": "ida" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "726573", "token_id": 417, "utf8_fragment": "res" }, { "bytes_hex": "756d656e", "token_id": 28999, "utf8_fragment": "umen" }, { "bytes_hex": "5f63", "token_id": 669, "utf8_fragment": "_c" }, { "bytes_hex": "6f7274", "token_id": 371, "utf8_fragment": "ort" }, { "bytes_hex": "6f", "token_id": 78, "utf8_fragment": "o" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "313230", "token_id": 4364, "utf8_fragment": "120" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "383530", "token_id": 16217, "utf8_fragment": "850" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "313830", "token_id": 5245, "utf8_fragment": "180" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "726573707565737461", "token_id": 58760, "utf8_fragment": "respuesta" }, { "bytes_hex": "5f636f6e", "token_id": 3464, "utf8_fragment": "_con" }, { "bytes_hex": "5f68", "token_id": 1552, "utf8_fragment": "_h" }, { "bytes_hex": "6973746f72", "token_id": 5436, "utf8_fragment": "istor" }, { "bytes_hex": "69616c", "token_id": 532, "utf8_fragment": "ial" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "363030", "token_id": 5067, "utf8_fragment": "600" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "343230", "token_id": 12819, "utf8_fragment": "420" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "323430", "token_id": 8273, "utf8_fragment": "240" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "363530", "token_id": 13655, "utf8_fragment": "650" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "616e616c", "token_id": 49921, "utf8_fragment": "anal" }, { "bytes_hex": "69736973", "token_id": 9667, "utf8_fragment": "isis" }, { "bytes_hex": "5f6465", "token_id": 2310, "utf8_fragment": "_de" }, { "bytes_hex": "5f746162", "token_id": 17796, "utf8_fragment": "_tab" }, { "bytes_hex": "6c61", "token_id": 4355, "utf8_fragment": "la" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "3930", "token_id": 1954, "utf8_fragment": "90" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "313835", "token_id": 9741, "utf8_fragment": "185" }, { "bytes_hex": "3030", "token_id": 410, "utf8_fragment": "00" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "323230", "token_id": 8610, "utf8_fragment": "220" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "7265766973696f6e", "token_id": 29442, "utf8_fragment": "revision" }, { "bytes_hex": "5f7265", "token_id": 1311, "utf8_fragment": "_re" }, { "bytes_hex": "706574", "token_id": 7005, "utf8_fragment": "pet" }, { "bytes_hex": "696461", "token_id": 4849, "utf8_fragment": "ida" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "333030", "token_id": 3101, "utf8_fragment": "300" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "373230", "token_id": 13104, "utf8_fragment": "720" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "363130", "token_id": 17608, "utf8_fragment": "610" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "393030", "token_id": 7467, "utf8_fragment": "900" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" } ], "sample_id": "tabla", "source_sha256": "9be1489ec451a5dbfa33c8ee2ad3cdc1e6b469dbf18385cdef2a3209aed5aa5d", "token_ids": [ 3380, 21044, 10900, 7918, 21237, 70259, 14287, 9912, 62, 63340, 14287, 9912, 11790, 14287, 9912, 63591, 4849, 198, 417, 28999, 669, 371, 78, 11, 4364, 15, 11, 16217, 11, 15, 11, 5245, 198, 58760, 3464, 1552, 5436, 532, 11, 5067, 11, 12819, 15, 11, 8273, 15, 11, 13655, 198, 49921, 9667, 2310, 17796, 4355, 11, 1954, 11, 9741, 410, 11, 15, 11, 8610, 15, 198, 29442, 1311, 7005, 4849, 11, 3101, 11, 13104, 15, 11, 17608, 15, 11, 7467, 198 ] }, { "encoding": "o200k_base", "pieces": [ { "bytes_hex": "657363", "token_id": 7484, "utf8_fragment": "esc" }, { "bytes_hex": "656e6172696f", "token_id": 28830, "utf8_fragment": "enario" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "736f6c", "token_id": 4399, "utf8_fragment": "sol" }, { "bytes_hex": "6963", "token_id": 291, "utf8_fragment": "ic" }, { "bytes_hex": "697475646573", "token_id": 25642, "utf8_fragment": "itudes" }, { "bytes_hex": "5f6d6573", "token_id": 181576, "utf8_fragment": "_mes" }, { "bytes_hex": "2c74", "token_id": 26927, "utf8_fragment": ",t" }, { "bytes_hex": "6f6b656e73", "token_id": 15515, "utf8_fragment": "okens" }, { "bytes_hex": "5f", "token_id": 62, "utf8_fragment": "_" }, { "bytes_hex": "656e7472616461", "token_id": 98567, "utf8_fragment": "entrada" }, { "bytes_hex": "2c74", "token_id": 26927, "utf8_fragment": ",t" }, { "bytes_hex": "6f6b656e73", "token_id": 15515, "utf8_fragment": "okens" }, { "bytes_hex": "5f6361636865", "token_id": 27045, "utf8_fragment": "_cache" }, { "bytes_hex": "2c74", "token_id": 26927, "utf8_fragment": ",t" }, { "bytes_hex": "6f6b656e73", "token_id": 15515, "utf8_fragment": "okens" }, { "bytes_hex": "5f73", "token_id": 1278, "utf8_fragment": "_s" }, { "bytes_hex": "616c696461", "token_id": 122452, "utf8_fragment": "alida" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "726573", "token_id": 430, "utf8_fragment": "res" }, { "bytes_hex": "756d656e", "token_id": 17512, "utf8_fragment": "umen" }, { "bytes_hex": "5f63", "token_id": 1303, "utf8_fragment": "_c" }, { "bytes_hex": "6f72746f", "token_id": 81641, "utf8_fragment": "orto" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "313230", "token_id": 6106, "utf8_fragment": "120" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "383530", "token_id": 24987, "utf8_fragment": "850" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "313830", "token_id": 7521, "utf8_fragment": "180" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "726573707565737461", "token_id": 143597, "utf8_fragment": "respuesta" }, { "bytes_hex": "5f636f6e", "token_id": 7531, "utf8_fragment": "_con" }, { "bytes_hex": "5f68697374", "token_id": 70597, "utf8_fragment": "_hist" }, { "bytes_hex": "6f7269616c", "token_id": 9466, "utf8_fragment": "orial" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "363030", "token_id": 6718, "utf8_fragment": "600" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "343230", "token_id": 22159, "utf8_fragment": "420" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "323430", "token_id": 13312, "utf8_fragment": "240" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "363530", "token_id": 20864, "utf8_fragment": "650" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "616e616c", "token_id": 15134, "utf8_fragment": "anal" }, { "bytes_hex": "69736973", "token_id": 11352, "utf8_fragment": "isis" }, { "bytes_hex": "5f6465", "token_id": 5047, "utf8_fragment": "_de" }, { "bytes_hex": "5f746162", "token_id": 38825, "utf8_fragment": "_tab" }, { "bytes_hex": "6c61", "token_id": 1675, "utf8_fragment": "la" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "3930", "token_id": 2744, "utf8_fragment": "90" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "313835", "token_id": 14741, "utf8_fragment": "185" }, { "bytes_hex": "3030", "token_id": 504, "utf8_fragment": "00" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "323230", "token_id": 12806, "utf8_fragment": "220" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" }, { "bytes_hex": "7265766973696f6e", "token_id": 64268, "utf8_fragment": "revision" }, { "bytes_hex": "5f7265", "token_id": 3067, "utf8_fragment": "_re" }, { "bytes_hex": "706574", "token_id": 9456, "utf8_fragment": "pet" }, { "bytes_hex": "696461", "token_id": 2261, "utf8_fragment": "ida" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "333030", "token_id": 4095, "utf8_fragment": "300" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "373230", "token_id": 21514, "utf8_fragment": "720" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "363130", "token_id": 31184, "utf8_fragment": "610" }, { "bytes_hex": "30", "token_id": 15, "utf8_fragment": "0" }, { "bytes_hex": "2c", "token_id": 11, "utf8_fragment": "," }, { "bytes_hex": "393030", "token_id": 9752, "utf8_fragment": "900" }, { "bytes_hex": "0a", "token_id": 198, "utf8_fragment": "\n" } ], "sample_id": "tabla", "source_sha256": "9be1489ec451a5dbfa33c8ee2ad3cdc1e6b469dbf18385cdef2a3209aed5aa5d", "token_ids": [ 7484, 28830, 11, 4399, 291, 25642, 181576, 26927, 15515, 62, 98567, 26927, 15515, 27045, 26927, 15515, 1278, 122452, 198, 430, 17512, 1303, 81641, 11, 6106, 15, 11, 24987, 11, 15, 11, 7521, 198, 143597, 7531, 70597, 9466, 11, 6718, 11, 22159, 15, 11, 13312, 15, 11, 20864, 198, 15134, 11352, 5047, 38825, 1675, 11, 2744, 11, 14741, 504, 11, 15, 11, 12806, 15, 198, 64268, 3067, 9456, 2261, 11, 4095, 11, 21514, 15, 11, 31184, 15, 11, 9752, 198 ] } ], "schema_version": "1.0.0", "tokenizer": { "package": "tiktoken", "version": "0.14.0" } }