Skip to content

vllm.entrypoints.scale_out.derender.api_router

Functions:

derender_chat_completion(request, raw_request) async

Derender a generate response into a ChatCompletionResponse.

Accepts both non-streaming (stream=false, default) and streaming (stream=true) request bodies on the same path; FastAPI validates and routes on the stream discriminator.

Non-streaming: body is DerenderChatRequest (generate_response with the complete token list). Returns a ChatCompletionResponse.

Streaming: body is DerenderChatStreamRequest (one generate_chunk delta + optional stream_state). Returns a DerenderChatStreamResponse (chunk + stream_state). The client carries stream_state between successive calls, one per SSE chunk from /inference/v1/generate.

Source code in vllm/entrypoints/scale_out/derender/api_router.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
@router.post(
    "/v1/chat/completions/derender",
    dependencies=[Depends(validate_json_request)],
    responses={
        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
    },
)
async def derender_chat_completion(
    request: DerenderChatRequestUnion,
    raw_request: Request,
):
    """Derender a generate response into a ChatCompletionResponse.

    Accepts both non-streaming (`stream=false`, default) and streaming
    (`stream=true`) request bodies on the same path; FastAPI validates and
    routes on the `stream` discriminator.

    Non-streaming: body is `DerenderChatRequest` (`generate_response` with
    the complete token list). Returns a `ChatCompletionResponse`.

    Streaming: body is `DerenderChatStreamRequest` (one `generate_chunk`
    delta + optional `stream_state`). Returns a `DerenderChatStreamResponse`
    (`chunk` + `stream_state`). The client carries `stream_state` between
    successive calls, one per SSE chunk from `/inference/v1/generate`.
    """
    handler = derender(raw_request)
    if handler is None:
        raise NotImplementedError(
            "The model does not support Chat Completions Derender API"
        )

    if isinstance(request, DerenderChatStreamRequest):
        stream_result = await handler.derender_chat_stream_response(request)
        if isinstance(stream_result, ErrorResponse):
            return JSONResponse(
                content=stream_result.model_dump(),
                status_code=stream_result.error.code,
            )
        chunk, stream_state = stream_result
        response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
        return JSONResponse(content=response.model_dump())

    result = await handler.derender_chat_response(request)
    if isinstance(result, ErrorResponse):
        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
    return JSONResponse(content=result.model_dump())

derender_completion(request, raw_request) async

Derender a generate response into a CompletionResponse.

Accepts both non-streaming (stream=false, default) and streaming (stream=true) request bodies on the same path.

Non-streaming: body is DerenderCompletionRequest. Returns a CompletionResponse.

Streaming: body is DerenderCompletionStreamRequest (one generate_chunk + optional stream_state). Returns a DerenderCompletionStreamResponse (chunk + stream_state).

Source code in vllm/entrypoints/scale_out/derender/api_router.py 
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
@router.post(
    "/v1/completions/derender",
    dependencies=[Depends(validate_json_request)],
    responses={
        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
    },
)
async def derender_completion(
    request: DerenderCompletionRequestUnion,
    raw_request: Request,
):
    """Derender a generate response into a CompletionResponse.

    Accepts both non-streaming (`stream=false`, default) and streaming
    (`stream=true`) request bodies on the same path.

    Non-streaming: body is `DerenderCompletionRequest`. Returns a
    `CompletionResponse`.

    Streaming: body is `DerenderCompletionStreamRequest` (one
    `generate_chunk` + optional `stream_state`). Returns a
    `DerenderCompletionStreamResponse` (`chunk` + `stream_state`).
    """
    handler = derender(raw_request)
    if handler is None:
        raise NotImplementedError("The model does not support Completions Derender API")

    if isinstance(request, DerenderCompletionStreamRequest):
        stream_result = await handler.derender_completion_stream_response(request)
        if isinstance(stream_result, ErrorResponse):
            return JSONResponse(
                content=stream_result.model_dump(),
                status_code=stream_result.error.code,
            )
        chunk, stream_state = stream_result
        response = DerenderCompletionStreamResponse(
            chunk=chunk, stream_state=stream_state
        )
        return JSONResponse(content=response.model_dump())

    result = await handler.derender_completion_response(request)
    if isinstance(result, ErrorResponse):
        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
    return JSONResponse(content=result.model_dump())