我已经训练并保存了一个模型。我试图在新数据上进一步训练模型，但它会产生错误。代码的相关部分:
from tensorflow.keras.preprocessing.text import Tokenizer
# The maximum number of words to be used. (most frequent)
MAX_NB_WORDS = 50000
# Max number of words in each complaint.
MAX_SEQUENCE_LENGTH = 250
# This is fixed.
EMBEDDING_DIM = 100
tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[]^_`{|}~', lower=True)
tokenizer.fit_on_texts(master_df['Observation'].values)
word_index = tokenizer.word_index
from sklearn.feature_extraction.text import CountVectorizer
cv=CountVectorizer(max_df=1.0,min_df=1, stop_words=stop_words, max_features=10000, ngram_range=(1,3))
X=cv.fit_transform(X)
with open("../sgd.pickle", 'rb') as f:
sgd = pickle.load(f)
def output_sample(sentence):
test=preprocess_text(sentence)
test=test.lower()
#print(test)
test=[test]
tokenizer.fit_on_sequences(test)
new_words= tokenizer.word_index
#print(word_index)``
test1=cv.transform(test)
#print(test1)
output=sgd.predict(test1)
return output[0]
def retrain(X,y):
X=preprocess_text(X)
X=X.lower()
X=[X]
tokenizer.fit_on_texts(X)
new_words=tokenizer.word_index
X=cv.fit_transform(X)
sgd.fit(X,y)
with open('sgd.pickle', 'wb') as f:
pickle.dump(sgd, f)
print("Model trained on new data")
sentence=input("nnEnter your observation:nn")
output=output_sample(sentence)
print("nnThe risk prediction is",preprocess_text(output),"nn")
print("Is the above prediction correct?n")
corr=input("Press 'y' for yes or 'n' for no.n")
if corr=='y':
newy=np.array(output)
retrain(sentence,newy)
elif corr=='n':
print("What is the correct risk?n1. Lown2. Mediumn")
r=input("Enter the appropriate number: ")
if r=='1':
newy=np.array('Low')
retrain(sentence,newy)
elif r=='2':
newy=np.array('Medium')
retrain(sentence,newy)
else:
print("Incorrect input. Please restart the application.")
else:
print("Incorrect input. Please restart the application")
程序运行时，错误发生在sgd.fit(X,y)。错误是
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
~AppDataLocalTemp/ipykernel_11300/3528077041.py in <module>
5 newy=[output]
6 print(newy)
----> 7 retrain(sentence,newy)
8
9 elif corr=='n':
~AppDataLocalTemp/ipykernel_11300/2433836763.py in retrain(X, y)
7 X=cv.fit_transform(X)
8 #y = y.reshape((-1, 1))
----> 9 sgd.fit(X,y)
10 with open('sgd.pickle', 'wb') as f:
11 pickle.dump(sgd, f)
~AppDataLocalProgramsPythonPython38libsite-packagessklearnpipeline.py in fit(self, X, y, **fit_params)
344 if self._final_estimator != 'passthrough':
345 fit_params_last_step = fit_params_steps[self.steps[-1][0]]
--> 346 self._final_estimator.fit(Xt, y, **fit_params_last_step)
347
348 return self
~AppDataLocalProgramsPythonPython38libsite-packagessklearnlinear_model_stochastic_gradient.py in fit(self, X, y, coef_init, intercept_init, sample_weight)
727 Returns an instance of self.
728 """
--> 729 return self._fit(X, y, alpha=self.alpha, C=1.0,
730 loss=self.loss, learning_rate=self.learning_rate,
731 coef_init=coef_init, intercept_init=intercept_init,
~AppDataLocalProgramsPythonPython38libsite-packagessklearnlinear_model_stochastic_gradient.py in _fit(self, X, y, alpha, C, loss, learning_rate, coef_init, intercept_init, sample_weight)
567 self.t_ = 1.0
568
--> 569 self._partial_fit(X, y, alpha, C, loss, learning_rate, self.max_iter,
570 classes, sample_weight, coef_init, intercept_init)
571
~AppDataLocalProgramsPythonPython38libsite-packagessklearnlinear_model_stochastic_gradient.py in _partial_fit(self, X, y, alpha, C, loss, learning_rate, max_iter, classes, sample_weight, coef_init, intercept_init)
529 max_iter=max_iter)
530 else:
--> 531 raise ValueError(
532 "The number of classes has to be greater than one;"
533 " got %d class" % n_classes)
ValueError: The number of classes has to be greater than one; got 1 class
数据示例如下:
Observation Risk
0 A separate road for light vehicle should be ma... Low
2 All benches were not having sufficient berm. Low
3 As light arrangement is not adequate. Low
4 As light arrangement is not adequate. Low
5 As contractor's equipment record is not availa... Low
77 First aid Room is not established. Medium
98 Heavy dust on haul road is found with in suffi... Medium
79 First aid station is maintained in the Rest sh... Medium
171 Presently explosive van is not available with ... Medium
79 First aid station is maintained in the Rest sh... Medium
理想情况下，它应该接受输入，但我不知道为什么它会给出这个错误。
我清理了代码并对retrain函数做了一些更改，现在该函数将添加一个新的字符串和标签到训练集并再次适合分类器。代码的其他部分在逻辑上保持不变!
效用函数:
def output_sample(sentence):
test=preprocess_text(sentence)
test=test.lower()
test=[test]
tokenizer.fit_on_sequences(test)
new_words= tokenizer.word_index
test1=cv.transform(test)
output=sgd.predict(test1)
return output[0]
def preprocess_text(string):
# do whatever you want but return String afterward ;)
return string
def retrain(X,y):
X=preprocess_text(X)
X=X.lower()
X=[X]
X = cv.fit_transform(master_df['Observation']+X)
new_words=tokenizer.word_index
sgd.fit(X,master_df['Risk']+y)
with open('sgd.pickle', 'wb') as f:
pickle.dump(sgd, f)
print("Model trained on new data")
import numpy as np
import pickle
import nltk
from sklearn.feature_extraction.text import CountVectorizer
stopwords = nltk.corpus.stopwords.words('english')
cv=CountVectorizer(max_df=1.0,min_df=1, stop_words=stopwords, max_features=10000, ngram_range=(1,3))
master_df = pd.read_csv('classification.tsv')
X=cv.fit_transform(master_df['Observation'])
from sklearn.linear_model import SGDClassifier
try:
f = open("./sgd.pickle", 'rb')
sgd = pickle.load(f)
except:
sgd = SGDClassifier()
sgd.fit(X, master_df['Risk'].to_list())
sentence=input("nnEnter your observation:nn")
output=output_sample(sentence)
print("nnThe risk prediction is",preprocess_text(output),"nn")
print("Is the above prediction correct?n")
corr=input("Press 'y' for yes or 'n' for no.n")
if corr=='y':
newy=np.array(output)
retrain(sentence, newy)
elif corr=='n':
print("What is the correct risk?n1. Lown2. Mediumn")
r=input("Enter the appropriate number: ")
if r=='1':
newy=np.array('Low')
retrain(sentence,newy)
elif r=='2':
newy=np.array('Medium')
retrain(sentence,newy)
else:
print("Incorrect input. Please restart the application.")
else:
print("Incorrect input. Please restart the application")