### Skip-Gram Model Implementation Code Below is an example of how a skip-gram model can be implemented using Python. This implementation initializes the necessary parameters and processes the input data into a format suitable for training. ```python import numpy as np class Word2VecSkipGram: def __init__(self, vocab_size, embedding_dim): self.vocab_size = vocab_size self.embedding_dim = embedding_dim # Initialize weights randomly self.W_in = np.random.randn(vocab_size, embedding_dim) # Input layer weight matrix self.W_out = np.random.randn(embedding_dim, vocab_size) # Output layer weight matrix def forward(self, center_word_one_hot): hidden_layer = np.dot(center_word_one_hot, self.W_in) output_layer = np.dot(hidden_layer, self.W_out) predicted_distribution = softmax(output_layer) return predicted_distribution, hidden_layer, center_word_one_hot def generate_training_data(corpus, window_size=2): """ Generates training samples from the given corpus. :param corpus: List[List[int]], where each sublist represents sentences with integer-encoded tokens. :param window_size: int, size of context window around target word. :return: Tuple containing lists of one-hot encoded vectors for both targets and contexts. """ train_x = [] train_y = [] for sentence in corpus: for i, token_id in enumerate(sentence): start_index = max(i - window_size, 0) end_index = min(i + window_size + 1, len(sentence)) for j in range(start_index, end_index): if j != i: train_x.append(to_one_hot(token_id)) train_y.append(to_one_hot(sentence[j])) return np.array(train_x), np.array(train_y) def to_one_hot(word_idx, vocab_size=None): """Converts index to one-hot vector.""" if not vocab_size: raise ValueError("Vocabulary size must be specified.") vec = np.zeros(vocab_size) vec[word_idx] = 1 return vec def softmax(x): e_x = np.exp(x - np.max(x)) # Subtracting max value improves numerical stability return e_x / e_x.sum(axis=-1, keepdims=True) # Example usage: vocab_size = 10000 # Size of vocabulary embedding_dim = 300 # Dimensionality of embeddings corpus = [[...]] # Preprocessed text dataset represented by indices w2v_sg_model = Word2VecSkipGram(vocab_size=vocab_size, embedding_dim=embedding_dim) train_inputs, train_labels = generate_training_data(corpus) for epoch in range(num_epochs): total_loss = 0 for idx in range(len(train_inputs)): prediction, h, u_o = w2v_sg_model.forward(train_inputs[idx]) loss = compute_loss(prediction, train_labels[idx]) # Define your own function here total_loss += loss.item() backpropagate_and_update_weights(loss, w2v_sg_model) # Implement these functions accordingly ``` This code snippet demonstrates initializing a `Word2VecSkipGram` class that handles creating random initializations for the embedding matrices used during training. The method `forward()` computes predictions based on current parameter values while also returning intermediate results needed later when updating gradients through backward propagation. Additionally, helper methods like converting integers representing words into their corresponding one-hot encodings or applying activation functions such as Softmax are included[^4].