All files / kernel-language-model-service/src/test-utils/queue service.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1x
 
 
3x
 
 
2x
 
 
3x
 
 
import type { QueueLanguageModel } from './model.ts';
import { makeQueueModel } from './model.ts';
import type { ObjectResponse, ResponseFormatter } from './response.ts';
import type { Tokenizer } from './tokenizer.ts';
import type { InstanceConfig, LanguageModelService } from '../../types.ts';
 
type QueueLanguageModelServiceConfig<Response extends object> = {
  tokenizer?: Tokenizer;
  responseFormatter?: ResponseFormatter<Response>;
};
 
/**
 * Queue-based language model service that returns QueueLanguageModel instances.
 * This is a minimal implementation of LanguageModelService that uses a queue for responses.
 *
 * @template Config - The type of configuration accepted by the service
 * @template Response - The type of response generated by created models
 */
export type QueueLanguageModelService<
  Response extends object = ObjectResponse,
> = LanguageModelService<
  QueueLanguageModelServiceConfig<Response>,
  unknown,
  Response
> & {
  /**
   * Creates a new queue-based language model instance.
   * The configuration is ignored - all instances use the 'test' model.
   *
   * @param config - The configuration for the model instance
   * @param config.tokenizer - The tokenizer function to use. Defaults to whitespace splitting.
   * @param config.responseFormatter - The function to use to format each yielded token into a response. Defaults to an object with a response and done property.
   * @returns A promise that resolves to a queue-based language model instance
   */
  makeInstance: (
    config: InstanceConfig<QueueLanguageModelServiceConfig<Response>>,
  ) => Promise<QueueLanguageModel<Response>>;
};
 
/**
 * Creates a queue-based language model service.
 * This is a minimal implementation of LanguageModelService that uses a queue for responses.
 *
 * @template Config - The type of configuration accepted by the service
 * @template Response - The type of response generated by created models
 * @returns A hardened queue-based language model service
 */
export const makeQueueService = <
  Response extends object = ObjectResponse,
>(): QueueLanguageModelService<Response> => {
  const makeInstance = async (
    config: InstanceConfig<QueueLanguageModelServiceConfig<Response>>,
  ): Promise<QueueLanguageModel<Response>> => {
    return makeQueueModel<Response>(config.options);
  };
 
  return harden({ makeInstance });
};