line.mjs 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145
  1. var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
  2. if (kind === "m") throw new TypeError("Private method is not writable");
  3. if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
  4. if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
  5. return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
  6. };
  7. var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
  8. if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
  9. if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
  10. return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
  11. };
  12. var _LineDecoder_carriageReturnIndex;
  13. import { OpenAIError } from "../../error.mjs";
  14. /**
  15. * A re-implementation of httpx's `LineDecoder` in Python that handles incrementally
  16. * reading lines from text.
  17. *
  18. * https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258
  19. */
  20. export class LineDecoder {
  21. constructor() {
  22. _LineDecoder_carriageReturnIndex.set(this, void 0);
  23. this.buffer = new Uint8Array();
  24. __classPrivateFieldSet(this, _LineDecoder_carriageReturnIndex, null, "f");
  25. }
  26. decode(chunk) {
  27. if (chunk == null) {
  28. return [];
  29. }
  30. const binaryChunk = chunk instanceof ArrayBuffer ? new Uint8Array(chunk)
  31. : typeof chunk === 'string' ? new TextEncoder().encode(chunk)
  32. : chunk;
  33. let newData = new Uint8Array(this.buffer.length + binaryChunk.length);
  34. newData.set(this.buffer);
  35. newData.set(binaryChunk, this.buffer.length);
  36. this.buffer = newData;
  37. const lines = [];
  38. let patternIndex;
  39. while ((patternIndex = findNewlineIndex(this.buffer, __classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f"))) != null) {
  40. if (patternIndex.carriage && __classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f") == null) {
  41. // skip until we either get a corresponding `\n`, a new `\r` or nothing
  42. __classPrivateFieldSet(this, _LineDecoder_carriageReturnIndex, patternIndex.index, "f");
  43. continue;
  44. }
  45. // we got double \r or \rtext\n
  46. if (__classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f") != null &&
  47. (patternIndex.index !== __classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f") + 1 || patternIndex.carriage)) {
  48. lines.push(this.decodeText(this.buffer.slice(0, __classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f") - 1)));
  49. this.buffer = this.buffer.slice(__classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f"));
  50. __classPrivateFieldSet(this, _LineDecoder_carriageReturnIndex, null, "f");
  51. continue;
  52. }
  53. const endIndex = __classPrivateFieldGet(this, _LineDecoder_carriageReturnIndex, "f") !== null ? patternIndex.preceding - 1 : patternIndex.preceding;
  54. const line = this.decodeText(this.buffer.slice(0, endIndex));
  55. lines.push(line);
  56. this.buffer = this.buffer.slice(patternIndex.index);
  57. __classPrivateFieldSet(this, _LineDecoder_carriageReturnIndex, null, "f");
  58. }
  59. return lines;
  60. }
  61. decodeText(bytes) {
  62. if (bytes == null)
  63. return '';
  64. if (typeof bytes === 'string')
  65. return bytes;
  66. // Node:
  67. if (typeof Buffer !== 'undefined') {
  68. if (bytes instanceof Buffer) {
  69. return bytes.toString();
  70. }
  71. if (bytes instanceof Uint8Array) {
  72. return Buffer.from(bytes).toString();
  73. }
  74. throw new OpenAIError(`Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`);
  75. }
  76. // Browser
  77. if (typeof TextDecoder !== 'undefined') {
  78. if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {
  79. this.textDecoder ?? (this.textDecoder = new TextDecoder('utf8'));
  80. return this.textDecoder.decode(bytes);
  81. }
  82. throw new OpenAIError(`Unexpected: received non-Uint8Array/ArrayBuffer (${bytes.constructor.name}) in a web platform. Please report this error.`);
  83. }
  84. throw new OpenAIError(`Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`);
  85. }
  86. flush() {
  87. if (!this.buffer.length) {
  88. return [];
  89. }
  90. return this.decode('\n');
  91. }
  92. }
  93. _LineDecoder_carriageReturnIndex = new WeakMap();
  94. // prettier-ignore
  95. LineDecoder.NEWLINE_CHARS = new Set(['\n', '\r']);
  96. LineDecoder.NEWLINE_REGEXP = /\r\n|[\n\r]/g;
  97. /**
  98. * This function searches the buffer for the end patterns, (\r or \n)
  99. * and returns an object with the index preceding the matched newline and the
  100. * index after the newline char. `null` is returned if no new line is found.
  101. *
  102. * ```ts
  103. * findNewLineIndex('abc\ndef') -> { preceding: 2, index: 3 }
  104. * ```
  105. */
  106. function findNewlineIndex(buffer, startIndex) {
  107. const newline = 0x0a; // \n
  108. const carriage = 0x0d; // \r
  109. for (let i = startIndex ?? 0; i < buffer.length; i++) {
  110. if (buffer[i] === newline) {
  111. return { preceding: i, index: i + 1, carriage: false };
  112. }
  113. if (buffer[i] === carriage) {
  114. return { preceding: i, index: i + 1, carriage: true };
  115. }
  116. }
  117. return null;
  118. }
  119. export function findDoubleNewlineIndex(buffer) {
  120. // This function searches the buffer for the end patterns (\r\r, \n\n, \r\n\r\n)
  121. // and returns the index right after the first occurrence of any pattern,
  122. // or -1 if none of the patterns are found.
  123. const newline = 0x0a; // \n
  124. const carriage = 0x0d; // \r
  125. for (let i = 0; i < buffer.length - 1; i++) {
  126. if (buffer[i] === newline && buffer[i + 1] === newline) {
  127. // \n\n
  128. return i + 2;
  129. }
  130. if (buffer[i] === carriage && buffer[i + 1] === carriage) {
  131. // \r\r
  132. return i + 2;
  133. }
  134. if (buffer[i] === carriage &&
  135. buffer[i + 1] === newline &&
  136. i + 3 < buffer.length &&
  137. buffer[i + 2] === carriage &&
  138. buffer[i + 3] === newline) {
  139. // \r\n\r\n
  140. return i + 4;
  141. }
  142. }
  143. return -1;
  144. }
  145. //# sourceMappingURL=line.mjs.map