Split stats in two and remove useless dimensions

This commit is contained in:
Jean-Marc Valin 2023-11-06 17:49:18 -05:00
parent 2386a60ec6
commit 0ab0640d4a
No known key found for this signature in database
GPG key ID: 531A52533318F00A
9 changed files with 98 additions and 78 deletions

View file

@ -124,6 +124,7 @@ def extract_diagonal(A):
return diag, B
def quantize_weight(weight, scale):
scale = scale + 1e-30
Aq = np.round(weight / scale).astype('int')
if Aq.max() > 127 or Aq.min() <= -128:
raise ValueError("value out of bounds in quantize_weight")
@ -227,7 +228,7 @@ def print_linear_layer(writer : CWriter,
nb_inputs, nb_outputs = weight.shape
if scale is None:
if scale is None and quantize:
scale = compute_scaling(weight)
@ -359,4 +360,4 @@ def print_gru_layer(writer : CWriter,
writer.header.write(f"\n#define {name.upper()}_OUT_SIZE {N}\n")
writer.header.write(f"\n#define {name.upper()}_STATE_SIZE {N}\n")
return N
return N