exllama
CUDA_VISIBLE_DEVICES=0 python -W ignore::UserWarning: test_benchmark_inference.py -d /data/ai/models/llm/llama/7b/llama-7b-hf -ppl gptq-for-llama -ppl-ds /data/ai/datasets/wikitext-2-raw/wiki.test.raw
GPTQ-for-LLaMA
Requires HF conversion of RAW files 5.677430152893066
CUDA_VISIBLE_DEVICES=0 python llama.py --wbits 16 --eval /data/ai/models/llm/llama/7b/llama-7b-hf wikitext2
Evaluating ...
0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
5.677430152893066
...
ptb
8.803622245788574
c4
7.078766822814941
# llama.cpp
We run our convert from the raw data to get an f16 unquantized version:
python convert.py /data/models-full/LLaMA/7B
context = 512
batch_size = 512
It calculates perplexity over 662 chunks and ends with: 6.3363
This is significantly different from the 5.9066 reported by https://github.com/ggerganov/llama.cpp#quantization
[lhl@zen2 llama.cpp]$ CUDA_VISIBLE_DEVICES=0 ./perplexity -m /data/ai/models/llm/llama/7b/ggml-model-f16.bin -ngl 99 -f /data/ai/datasets/wikitext-2/wiki.test.tokens main: build = 689 (d411968) main: seed = 1687013232 ggml_init_cublas: found 1 CUDA devices: Device 0: NVIDIA GeForce RTX 4090 llama.cpp: loading model from /data/ai/models/llm/llama/7b/ggml-model-f16.bin llama_model_load_internal: format = ggjt v1 (pre #1405) llama_model_load_internal: n_vocab = 32000 llama_model_load_internal: n_ctx = 512 llama_model_load_internal: n_embd = 4096 llama_model_load_internal: n_mult = 256 llama_model_load_internal: n_head = 32 llama_model_load_internal: n_layer = 32 llama_model_load_internal: n_rot = 128 llama_model_load_internal: ftype = 1 (mostly F16) llama_model_load_internal: n_ff = 11008 llama_model_load_internal: n_parts = 1 llama_model_load_internal: model size = 7B llama_model_load_internal: ggml ctx size = 0.07 MB llama_model_load_internal: using CUDA for GPU acceleration llama_model_load_internal: mem required = 2042.07 MB (+ 1026.00 MB per state) llama_model_load_internal: allocating batch_size x 1 MB = 512 MB VRAM for the scratch buffer llama_model_load_internal: offloading 32 repeating layers to GPU llama_model_load_internal: offloading non-repeating layers to GPU llama_model_load_internal: offloading v cache to GPU llama_model_load_internal: offloading k cache to GPU llama_model_load_internal: offloaded 35/35 layers to GPU llama_model_load_internal: total VRAM used: 14142 MB … llama_init_from_file: kv self size = 256.00 MB
system_info: n_threads = 16 / 32 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | VSX = 0 | perplexity: calculating perplexity over 662 chunks, batch_size=512 perplexity: 0.35 seconds per pass - ETA 3 minutes [1]4.4503,[2]4.9572,[3]5.7518,[4]6.4349,[5]6.7060,[6]6.7873,[7]6.9685,[8]7.0805,[9]7.3293,[10]7.5390,[11]7.6458,[12]7.7110,[13]7.6671,[14]7.5352,[15]7.8020,[16]7.4268,[17]7.3306,[18]7.2740,[19]6.8945,[20]6.9594,[21]6.9933,[22]6.8045,[23]6.7735,[24]6.6408,[25]6.5810,[26]6.4422,[27]6.2495,[28]6.1320,[29]6.0203,[30]5.8667,[31]5.8186,[32]5.8001,[33]5.7451,[34]5.7762,[35]5.8011,[36]5.8398,[37]5.7969,[38]5.8041,[39]5.8077,[40]5.8092,[41]5.8438,[42]5.8648,[43]5.8339,[44]5.8496,[45]5.8730,[46]5.8727,[47]5.8610,[48]5.8968,[49]5.9053,[50]5.8554,[51]5.8389,[52]5.7840,[53]5.8189,[54]5.8575,[55]5.8604,[56]5.8726,[57]5.9155,[58]5.9214,[59]5.9517,[60]5.9937,[61]6.0120,[62]6.0516,[63]6.0889,[64]6.0930,[65]6.1076,[66]6.1250,[67]6.1395,[68]6.1621,[69]6.1501,[70]6.1642,[71]6.1849,[72]6.2282,[73]6.2607,[74]6.2842,[75]6.3218,[76]6.3125,[77]6.3215,[78]6.3263,[79]6.3190,[80]6.3507,[81]6.3521,[82]6.3456,[83]6.3388,[84]6.3133,[85]6.3057,[86]6.3060,[87]6.2936,[88]6.2270,[89]6.2118,[90]6.1901,[91]6.1841,[92]6.2017,[93]6.1981,[94]6.2029,[95]6.2025,[96]6.2333,[97]6.2329,[98]6.2331,[99]6.2318,[100]6.2343,[101]6.2246,[102]6.2526,[103]6.2639,[104]6.2691,[105]6.2857,[106]6.2958,[107]6.3146,[108]6.3129,[109]6.3076,[110]6.3134,[111]6.3101,[112]6.3434,[113]6.3779,[114]6.4007,[115]6.3955,[116]6.4079,[117]6.3972,[118]6.4065,[119]6.4040,[120]6.4396,[121]6.4596,[122]6.5000,[123]6.5098,[124]6.5411,[125]6.5486,[126]6.5663,[127]6.5597,[128]6.5879,[129]6.6156,[130]6.6206,[131]6.6089,[132]6.5968,[133]6.5978,[134]6.5997,[135]6.6044,[136]6.6057,[137]6.5960,[138]6.5818,[139]6.5748,[140]6.5658,[141]6.5662,[142]6.5267,[143]6.4991,[144]6.4833,[145]6.4588,[146]6.4536,[147]6.4564,[148]6.4564,[149]6.4567,[150]6.4627,[151]6.4615,[152]6.4567,[153]6.4357,[154]6.4461,[155]6.4442,[156]6.4463,[157]6.4620,[158]6.4662,[159]6.4353,[160]6.4570,[161]6.4347,[162]6.4153,[163]6.3929,[164]6.3682,[165]6.3507,[166]6.3144,[167]6.3027,[168]6.2861,[169]6.2730,[170]6.2622,[171]6.2436,[172]6.2435,[173]6.2245,[174]6.2079,[175]6.1880,[176]6.1701,[177]6.1479,[178]6.1318,[179]6.1221,[180]6.1036,[181]6.0837,[182]6.0805,[183]6.0665,[184]6.0672,[185]6.0581,[186]6.0592,[187]6.0552,[188]6.0619,[189]6.0751,[190]6.0889,[191]6.1085,[192]6.1336,[193]6.1494,[194]6.1816,[195]6.1992,[196]6.2157,[197]6.2417,[198]6.2641,[199]6.2728,[200]6.2789,[201]6.2786,[202]6.2888,[203]6.2965,[204]6.2992,[205]6.3062,[206]6.3116,[207]6.3122,[208]6.3233,[209]6.3239,[210]6.3278,[211]6.3363,[212]6.3483,[213]6.3582,[214]6.3630,[215]6.3671,[216]6.3779,[217]6.3954,[218]6.4079,[219]6.4086,[220]6.4060,[221]6.3997,[222]6.3981,[223]6.3883,[224]6.3812,[225]6.3787,[226]6.3950,[227]6.3939,[228]6.4031,[229]6.4108,[230]6.4054,[231]6.4150,[232]6.3981,[233]6.3758,[234]6.3800,[235]6.3748,[236]6.3761,[237]6.3789,[238]6.3728,[239]6.3598,[240]6.3538,[241]6.3567,[242]6.3428,[243]6.3349,[244]6.3238,[245]6.3081,[246]6.2895,[247]6.2835,[248]6.2796,[249]6.2720,[250]6.2684,[251]6.2622,[252]6.2434,[253]6.2349,[254]6.2306,[255]6.2140,[256]6.2057,[257]6.1915,[258]6.1809,[259]6.1647,[260]6.1554,[261]6.1530,[262]6.1531,[263]6.1472,[264]6.1323,[265]6.1313,[266]6.1294,[267]6.1271,[268]6.1231,[269]6.1289,[270]6.1274,[271]6.1361,[272]6.1372,[273]6.1340,[274]6.1387,[275]6.1442,[276]6.1483,[277]6.1639,[278]6.1688,[279]6.1737,[280]6.1751,[281]6.1812,[282]6.1833,[283]6.1907,[284]6.1979,[285]6.2105,[286]6.2119,[287]6.2136,[288]6.2154,[289]6.2094,[290]6.2042,[291]6.1952,[292]6.1904,[293]6.1785,[294]6.1709,[295]6.1652,[296]6.1619,[297]6.1652,[298]6.1652,[299]6.1641,[300]6.1610,[301]6.1547,[302]6.1538,[303]6.1493,[304]6.1435,[305]6.1350,[306]6.1341,[307]6.1309,[308]6.1277,[309]6.1266,[310]6.1185,[311]6.1157,[312]6.1101,[313]6.1103,[314]6.1100,[315]6.1069,[316]6.1055,[317]6.1006,[318]6.0934,[319]6.0780,[320]6.0635,[321]6.0713,[322]6.0691,[323]6.0709,[324]6.0744,[325]6.0748,[326]6.0779,[327]6.0857,[328]6.0851,[329]6.0868,[330]6.0930,[331]6.0997,[332]6.1057,[333]6.1176,[334]6.1174,[335]6.1235,[336]6.1177,[337]6.1141,[338]6.1152,[339]6.1126,[340]6.1142,[341]6.1092,[342]6.1068,[343]6.1136,[344]6.1157,[345]6.1196,[346]6.1216,[347]6.1221,[348]6.1229,[349]6.1227,[350]6.1288,[351]6.1348,[352]6.1314,[353]6.1301,[354]6.1365,[355]6.1341,[356]6.1387,[357]6.1444,[358]6.1476,[359]6.1466,[360]6.1538,[361]6.1596,[362]6.1516,[363]6.1603,[364]6.1630,[365]6.1690,[366]6.1686,[367]6.1780,[368]6.1792,[369]6.1810,[370]6.1842,[371]6.1809,[372]6.1796,[373]6.1808,[374]6.1775,[375]6.1813,[376]6.1838,[377]6.1823,[378]6.1766,[379]6.1746,[380]6.1764,[381]6.1836,[382]6.1824,[383]6.1840,[384]6.1782,[385]6.1730,[386]6.1736,[387]6.1702,[388]6.1732,[389]6.1709,[390]6.1688,[391]6.1655,[392]6.1633,[393]6.1583,[394]6.1536,[395]6.1453,[396]6.1441,[397]6.1447,[398]6.1417,[399]6.1388,[400]6.1454,[401]6.1486,[402]6.1512,[403]6.1456,[404]6.1492,[405]6.1501,[406]6.1554,[407]6.1589,[408]6.1575,[409]6.1546,[410]6.1577,[411]6.1582,[412]6.1691,[413]6.1790,[414]6.1892,[415]6.2000,[416]6.2114,[417]6.2210,[418]6.2285,[419]6.2372,[420]6.2459,[421]6.2492,[422]6.2552,[423]6.2654,[424]6.2724,[425]6.2761,[426]6.2830,[427]6.2919,[428]6.3014,[429]6.3135,[430]6.3207,[431]6.3259,[432]6.3320,[433]6.3419,[434]6.3535,[435]6.3547,[436]6.3538,[437]6.3510,[438]6.3536,[439]6.3536,[440]6.3645,[441]6.3714,[442]6.3744,[443]6.3810,[444]6.3764,[445]6.3709,[446]6.3734,[447]6.3780,[448]6.3803,[449]6.3800,[450]6.3800,[451]6.3834,[452]6.3891,[453]6.3909,[454]6.3893,[455]6.3868,[456]6.3780,[457]6.3760,[458]6.3716,[459]6.3756,[460]6.3804,[461]6.3812,[462]6.3791,[463]6.3804,[464]6.3872,[465]6.3867,[466]6.3839,[467]6.3873,[468]6.3896,[469]6.3888,[470]6.3851,[471]6.3886,[472]6.3928,[473]6.3961,[474]6.3946,[475]6.3912,[476]6.3937,[477]6.3911,[478]6.3932,[479]6.3951,[480]6.3973,[481]6.3971,[482]6.3966,[483]6.3955,[484]6.3987,[485]6.4000,[486]6.4037,[487]6.4029,[488]6.4016,[489]6.4032,[490]6.4010,[491]6.4002,[492]6.4009,[493]6.3974,[494]6.3960,[495]6.3995,[496]6.3946,[497]6.3892,[498]6.3901,[499]6.3828,[500]6.3817,[501]6.3801,[502]6.3755,[503]6.3744,[504]6.3696,[505]6.3707,[506]6.3690,[507]6.3658,[508]6.3655,[509]6.3584,[510]6.3576,[511]6.3595,[512]6.3521,[513]6.3482,[514]6.3468,[515]6.3494,[516]6.3479,[517]6.3493,[518]6.3513,[519]6.3566,[520]6.3534,[521]6.3511,[522]6.3516,[523]6.3518,[524]6.3559,[525]6.3575,[526]6.3594,[527]6.3621,[528]6.3625,[529]6.3672,[530]6.3727,[531]6.3734,[532]6.3766,[533]6.3728,[534]6.3749,[535]6.3717,[536]6.3697,[537]6.3708,[538]6.3740,[539]6.3685,[540]6.3642,[541]6.3576,[542]6.3561,[543]6.3594,[544]6.3653,[545]6.3672,[546]6.3649,[547]6.3628,[548]6.3652,[549]6.3651,[550]6.3637,[551]6.3638,[552]6.3584,[553]6.3570,[554]6.3553,[555]6.3559,[556]6.3523,[557]6.3524,[558]6.3502,[559]6.3451,[560]6.3384,[561]6.3338,[562]6.3322,[563]6.3297,[564]6.3298,[565]6.3263,[566]6.3267,[567]6.3249,[568]6.3199,[569]6.3231,[570]6.3264,[571]6.3247,[572]6.3269,[573]6.3219,[574]6.3170,[575]6.3166,[576]6.3169,[577]6.3191,[578]6.3187,[579]6.3214,[580]6.3189,[581]6.3173,[582]6.3169,[583]6.3141,[584]6.3104,[585]6.3087,[586]6.3011,[587]6.2978,[588]6.2961,[589]6.2949,[590]6.2945,[591]6.2906,[592]6.2851,[593]6.2867,[594]6.2879,[595]6.2899,[596]6.2965,[597]6.2971,[598]6.2977,[599]6.2951,[600]6.2963,[601]6.2943,[602]6.2978,[603]6.2988,[604]6.2990,[605]6.3025,[606]6.3068,[607]6.3074,[608]6.3072,[609]6.3096,[610]6.3139,[611]6.3159,[612]6.3212,[613]6.3239,[614]6.3156,[615]6.3115,[616]6.3140,[617]6.3147,[618]6.3148,[619]6.3139,[620]6.3131,[621]6.3070,[622]6.3080,[623]6.3073,[624]6.3018,[625]6.2959,[626]6.2900,[627]6.2772,[628]6.2742,[629]6.2741,[630]6.2785,[631]6.2809,[632]6.2818,[633]6.2848,[634]6.2865,[635]6.2868,[636]6.2882,[637]6.2905,[638]6.2959,[639]6.2992,[640]6.3037,[641]6.3076,[642]6.3098,[643]6.3092,[644]6.3086,[645]6.3089,[646]6.3090,[647]6.3110,[648]6.3114,[649]6.3166,[650]6.3163,[651]6.3187,[652]6.3179,[653]6.3229,[654]6.3230,[655]6.3240,[656]6.3240,[657]6.3277,[658]6.3334,[659]6.3361,[660]6.3388,[661]6.3324,[662]6.3363,
llama_print_timings: load time = 2528.36 ms llama_print_timings: sample time = 0.00 ms / 1 runs ( 0.00 ms per token) llama_print_timings: prompt eval time = 221400.24 ms / 338944 tokens ( 0.65 ms per token) llama_print_timings: eval time = 0.00 ms / 1 runs ( 0.00 ms per token) llama_print_timings: total time = 245460.55 ms