Transformers.js Streaming

async () => {
  // Create a text generation pipeline
  const generator = await pipeline('text-generation', 'onnx-community/Qwen2.5-Coder-0.5B-Instruct', { dtype: 'q4' })

  // Define the list of messages
  const messages = [
    { role: 'system', content: 'You are a helpful assistant.' },
    { role: 'user', content: 'Write a quick sort algorithm.' }
  ]

  // Create text streamer
  const streamer = new TextStreamer(generator.tokenizer, {
    skip_prompt: true
  })

  // Generate a response
  const output = await generator(messages, { max_new_tokens: 500, do_sample: false, streamer })
  const content = ((output[0] as TextGenerationSingle).generated_text.at(-1) as Message).content
  console.info(content)
}

async () => {
    const pipe = await pipeline('text-generation', 'Xenova/LiteLlama-460M-1T', {
      dtype: 'q8',
      model_file_name: 'decoder_model_merged'
    })
    let response = ''
    // Create text streamer
    const streamer = new TextStreamer(pipe.tokenizer, {
      skip_prompt: true,
      callback_function: (text: string) => {
        response += text
        process.stdout.write(text)
      }
    })

    const output = await pipe(
      `### Context: General Relativity and Special Relativity are two main topics in Relative Mechanics.
Einstein Field Equatioqns is mathmetical model for General Relativity
### Question: What is Relative Mechanics?
### Response:`,
      {
        max_length: 75,
        // @ts-ignore
        streamer: streamer
      }
    )
    console.info(output)
    expect(response.indexOf('Relativity') < 2).toBe(true)
    console.info(response)
  }