model=/mnt/data/models/ubergarm/Step-3.5-Flash-GGUF/Step-3.5-Flash-Q8_0.gguf numactl -N "$SOCKET" -m "$SOCKET" \ ./build/bin/llama-server \ --model "$model"\ --alias ubergarm/Step-3.5-Flash \ --ctx-size 65536 \ -ctk q8_0 -ctv q8_0 \ -ub 4096 -b 4096 \ --parallel 1 \ --threads 96 \ --threads-batch 128 \ --numa numactl \ --host 127.0.0.1 \ --port 8080 \ --no-mmap \ --jinja \ --validate-quants INFO [ main] build info | tid="133504258449024" timestamp=1770404877 build=4183 commit="9a0b5e80" INFO [ main] system info | tid="133504258449024" timestamp=1770404877 n_threads=96 n_threads_batch=128 total_threads=512 system_info="AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | " CPU: using device CPU - 0 MiB free llama_model_loader: loaded meta data with 47 key-value pairs and 754 tensors from /mnt/data/models/ubergarm/Step-3.5-Flash-GGUF/Step-3.5-Flash-Q8_0.gguf (version GGUF V3 (latest)) llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. llama_model_loader: - kv 0: general.architecture str = step35 llama_model_loader: - kv 1: general.type str = model llama_model_loader: - kv 2: general.name str = Step 3.5 Flash llama_model_loader: - kv 3: general.size_label str = 288x7.4B llama_model_loader: - kv 4: general.license str = apache-2.0 llama_model_loader: - kv 5: general.base_model.count u32 = 1 llama_model_loader: - kv 6: general.base_model.0.name str = Step 3.5 Flash llama_model_loader: - kv 7: general.base_model.0.organization str = Stepfun Ai llama_model_loader: - kv 8: general.base_model.0.repo_url str = https://huggingface.co/stepfun-ai/ste... llama_model_loader: - kv 9: step35.block_count u32 = 45 llama_model_loader: - kv 10: step35.context_length u32 = 262144 llama_model_loader: - kv 11: step35.embedding_length u32 = 4096 llama_model_loader: - kv 12: step35.feed_forward_length u32 = 11264 llama_model_loader: - kv 13: step35.attention.head_count arr[i32,45] = [64, 96, 96, 96, 64, 96, 96, 96, 64, ... llama_model_loader: - kv 14: step35.rope.freq_base f32 = 5000000.000000 llama_model_loader: - kv 15: step35.rope.freq_base_swa f32 = 10000.000000 llama_model_loader: - kv 16: step35.expert_gating_func u32 = 2 llama_model_loader: - kv 17: step35.attention.key_length u32 = 128 llama_model_loader: - kv 18: step35.attention.value_length u32 = 128 llama_model_loader: - kv 19: general.file_type u32 = 7 llama_model_loader: - kv 20: step35.attention.head_count_kv arr[i32,45] = [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, ... llama_model_loader: - kv 21: step35.attention.sliding_window u32 = 512 llama_model_loader: - kv 22: step35.attention.sliding_window_pattern arr[i32,45] = [0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, ... llama_model_loader: - kv 23: step35.expert_count u32 = 288 llama_model_loader: - kv 24: step35.expert_used_count u32 = 8 llama_model_loader: - kv 25: step35.expert_feed_forward_length u32 = 1280 llama_model_loader: - kv 26: step35.expert_shared_feed_forward_length u32 = 1280 llama_model_loader: - kv 27: step35.expert_weights_scale f32 = 3.000000 llama_model_loader: - kv 28: step35.expert_weights_norm bool = true llama_model_loader: - kv 29: step35.leading_dense_block_count u32 = 3 llama_model_loader: - kv 30: step35.moe_every_n_layers u32 = 1 llama_model_loader: - kv 31: step35.attention.layer_norm_rms_epsilon f32 = 0.000010 llama_model_loader: - kv 32: step35.swiglu_clamp_exp arr[f32,45] = [0.000000, 0.000000, 0.000000, 0.0000... llama_model_loader: - kv 33: step35.swiglu_clamp_shexp arr[f32,45] = [0.000000, 0.000000, 0.000000, 0.0000... llama_model_loader: - kv 34: general.quantization_version u32 = 2 llama_model_loader: - kv 35: tokenizer.ggml.model str = gpt2 llama_model_loader: - kv 36: tokenizer.ggml.pre str = deepseek-v3 llama_model_loader: - kv 37: tokenizer.ggml.tokens arr[str,128896] = ["<｜begin▁of▁sentence｜>", "<... llama_model_loader: - kv 38: tokenizer.ggml.token_type arr[i32,128896] = [3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... llama_model_loader: - kv 39: tokenizer.ggml.merges arr[str,127741] = ["Ġ t", "Ġ a", "i n", "Ġ Ġ", "h e... llama_model_loader: - kv 40: tokenizer.ggml.bos_token_id u32 = 0 llama_model_loader: - kv 41: tokenizer.ggml.eos_token_id u32 = 128007 llama_model_loader: - kv 42: tokenizer.ggml.padding_token_id u32 = 1 llama_model_loader: - kv 43: tokenizer.ggml.add_bos_token bool = true llama_model_loader: - kv 44: tokenizer.ggml.add_sep_token bool = false llama_model_loader: - kv 45: tokenizer.ggml.add_eos_token bool = false llama_model_loader: - kv 46: tokenizer.chat_template str = {% macro render_content(content) %}{%... llama_model_loader: - type f32: 266 tensors llama_model_loader: - type q8_0: 488 tensors load: printing all EOG tokens: load: - 128007 ('<|im_end|>') load: special tokens cache size = 818 load: token to piece cache size = 0.8220 MB llm_load_print_meta: format = GGUF V3 (latest) llm_load_print_meta: arch = step35 llm_load_print_meta: n_ctx_train = 262144 llm_load_print_meta: n_embd = 4096 llm_load_print_meta: n_layer = 45 llm_load_print_meta: n_head = [64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64, 96, 96, 96, 64] llm_load_print_meta: n_head_kv = 8 llm_load_print_meta: n_rot = 128 llm_load_print_meta: n_swa = 512 llm_load_print_meta: n_swa_pattern = 1 llm_load_print_meta: n_embd_head_k = 128 llm_load_print_meta: n_embd_head_v = 128 llm_load_print_meta: n_gqa = [8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8, 12, 12, 12, 8] llm_load_print_meta: n_embd_k_gqa = 1024 llm_load_print_meta: n_embd_v_gqa = 1024 llm_load_print_meta: f_norm_eps = 0.0e+00 llm_load_print_meta: f_norm_rms_eps = 1.0e-05 llm_load_print_meta: f_clamp_kqv = 0.0e+00 llm_load_print_meta: f_max_alibi_bias = 0.0e+00 llm_load_print_meta: f_logit_scale = 0.0e+00 llm_load_print_meta: n_ff = 11264 llm_load_print_meta: n_expert = 288 llm_load_print_meta: n_expert_used = 8 llm_load_print_meta: causal attn = 1 llm_load_print_meta: pooling type = 0 llm_load_print_meta: rope type = 2 llm_load_print_meta: rope scaling = linear llm_load_print_meta: freq_base_train = 5000000.0 llm_load_print_meta: freq_scale_train = 1 llm_load_print_meta: n_ctx_orig_yarn = 262144 llm_load_print_meta: rope_finetuned = unknown llm_load_print_meta: ssm_d_conv = 0 llm_load_print_meta: ssm_d_inner = 0 llm_load_print_meta: ssm_d_state = 0 llm_load_print_meta: ssm_dt_rank = 0 llm_load_print_meta: model type = ?B llm_load_print_meta: model ftype = Q8_0 llm_load_print_meta: model params = 196.956 B llm_load_print_meta: model size = 195.031 GiB (8.506 BPW) llm_load_print_meta: repeating layers = 193.986 GiB (8.506 BPW, 195.900 B parameters) llm_load_print_meta: general.name = Step 3.5 Flash print_info: vocab type = BPE print_info: n_vocab = 128896 print_info: n_merges = 127741 print_info: BOS token = 0 '<｜begin▁of▁sentence｜>' print_info: EOS token = 128007 '<|im_end|>' print_info: EOT token = 128007 '<|im_end|>' print_info: PAD token = 1 '<｜end▁of▁sentence｜>' print_info: LF token = 201 'Ċ' print_info: FIM PRE token = 128801 '<｜fim▁begin｜>' print_info: FIM SUF token = 128800 '<｜fim▁hole｜>' print_info: FIM MID token = 128802 '<｜fim▁end｜>' print_info: EOG token = 128007 '<|im_end|>' print_info: max token length = 256 llm_load_tensors: ggml ctx size = 0.31 MiB llm_load_tensors: offloading 0 repeating layers to GPU llm_load_tensors: offloaded 0/46 layers to GPU llm_load_tensors: CPU buffer size = 199711.42 MiB .................................................................................................... llama_new_context_with_model: n_ctx = 65536 llama_new_context_with_model: n_batch = 4096 llama_new_context_with_model: n_ubatch = 4096 llama_new_context_with_model: flash_attn = 1 llama_new_context_with_model: attn_max_b = 0 llama_new_context_with_model: fused_moe = 1 llama_new_context_with_model: grouped er = 0 llama_new_context_with_model: fused_up_gate = 1 llama_new_context_with_model: fused_mmad = 1 llama_new_context_with_model: rope_cache = 0 llama_new_context_with_model: graph_reuse = 1 llama_new_context_with_model: k_cache_hadam = 0 llama_new_context_with_model: split_mode_graph_scheduling = 0 llama_new_context_with_model: reduce_type = f16 llama_new_context_with_model: sched_async = 0 llama_new_context_with_model: ser = -1, 0 llama_new_context_with_model: freq_base = 5000000.0 llama_new_context_with_model: freq_scale = 1 llama_kv_cache_init: CPU KV buffer size = 6120.00 MiB llama_new_context_with_model: KV self size = 6120.00 MiB, K (q8_0): 3060.00 MiB, V (q8_0): 3060.00 MiB llama_new_context_with_model: CPU output buffer size = 0.49 MiB llama_new_context_with_model: CPU compute buffer size = 2078.00 MiB llama_new_context_with_model: graph nodes = 2201 llama_new_context_with_model: graph splits = 1 XXXXXXXXXXXXXXXXXXXXX Setting only active experts offload ===================================== llama_new_context_with_model: f16 ======================================= HAVE_FANCY_SIMD is defined INFO [ init] initializing slots | tid="133504258449024" timestamp=1770404919 n_slots=1 INFO [ init] new slot | tid="133504258449024" timestamp=1770404919 id_slot=0 n_ctx_slot=65536 srv init: Exclude reasoning tokens when selecting slot based on similarity: start: , end: use `--reasoning-tokens none` to disable. prompt cache is enabled, size limit: 8192 MiB use `--cache-ram 0` to disable the prompt cache INFO [ main] model loaded | tid="133504258449024" timestamp=1770404919 INFO [ main] chat template | tid="133504258449024" timestamp=1770404919 chat_template="{% macro render_content(content) %}{% if content is none %}{{- '' }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}{% endif %}{% endfor %}{% endif %}{% endmacro %}\n{{bos_token}}{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- render_content(messages[0].content) + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions in JSONSchema format:\\n\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson(ensure_ascii=False) }}\n {%- endfor %}\n {{- \"\\n\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner \\n...\\n block must be nested within \\n...\\n XML tags\\n- Required parameters MUST be specified\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + render_content(messages[0].content) + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and render_content(message.content) is string and not(render_content(message.content).startswith('') and render_content(message.content).endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- set content = render_content(message.content) %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {%- set role_name = 'observation' if (message.role == \"system\" and not loop.first and message.name == 'observation') else message.role %}\n {{- '<|im_start|>' + role_name + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = render_content(message.reasoning_content) %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- else %}\n {%- set reasoning_content = '' %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- if tool_call.arguments is defined %}\n {%- set arguments = tool_call.arguments %}\n {%- for args_name, args_value in arguments|items %}\n {{- '\\n' }}\n {%- set args_value = args_value | tojson(ensure_ascii=False) | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>tool_response\\n' }}\n {%- endif %}\n {{- '' }}\n {{- content }}\n {{- '' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n\\n' }}\n{%- endif %}\n" INFO [ main] chat template | tid="133504258449024" timestamp=1770404919 chat_example="<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nHow are you?<|im_end|>\n<|im_start|>assistant\n\n" built_in=true INFO [ main] HTTP server listening | tid="133504258449024" timestamp=1770404919 n_threads_http="511" port="8080" hostname="127.0.0.1" INFO [ slots_idle] all slots are idle | tid="133504258449024" timestamp=1770404919