使用神经网络为图像生成标题（上）

import tensorflow as tf
from keras.preprocessing import image
import numpy as np
# function to extract features from image
def extract_image_features():
    model = tf.keras.models.Sequential()
    # adding first layers of convolution and pooling layers to network
    model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), input_shape=(90,90,3), padding="VALID", activation="relu"))
    model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"))
    model.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))
    # adding second layers of convolution and pooling layers to network
    model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=(3,3), padding="VALID", activation="relu"))
    model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu"))
    model.add(tf.keras.layers.AveragePooling2D(pool_size=2, strides=1))
    # flattening the output using flatten layer, since the input to neural net has to be flat
    model.add(tf.keras.layers.Flatten())
    # model summary
    model.summary()
    return model
for file in os.listdir(image_path):
    path = image_path + "//" + file
    img = image.load_img(path, target_size=(90, 90))
    img_data = image.img_to_array(img)
    img_data = np.expand_dims(img_data, axis=0)
    img_data = preprocess_input(img_data)
    feature = extract_image_features.predict(img_data)
    feature = np.reshape(feature, feature.shape[1])

import tensorflow as tf
from keras.preprocessing import image
from keras.applications.resnet50 import ResNet50
from keras.applications.resnet50 import preprocess_input
from keras.models import Model
# load the ResNet50 Model
feature_extractor = ResNet50(weights='imagenet', include_top=False)
feature_extractor_new = Model(feature_extractor.input, feature_extractor.layers[-2].output)
feature_extractor_new.summary()
for file in os.listdir(image_path):
    path = image_path + "//" + file
    img = image.load_img(path, target_size=(90, 90))
    img_data = image.img_to_array(img)
    img_data = np.expand_dims(img_data, axis=0)
    img_data = preprocess_input(img_data)
    feature = feature_extractor_new.predict(img_data)
    feature_reshaped = np.array(feature).flatten()

image_feature_dictionary[list(image_feature_dictionary. Keys())[0]].shape
(18432,)

# loading captions from captions file
import pandas as pd
# loading captions.txt
captions = pd.read_csv('/kaggle/input/flickr8k/captions.txt', sep=",")
captions = captions.rename(columns=lambda x: x.strip().lower())
captions['image'] = captions['image'].apply(lambda x: x.split(".")[0])
captions = captions[['image', 'caption']]
# adding <start> and <end> to every caption
captions['caption'] = "<start> " + captions['caption'] + " <end>"
# in case we have any missing caption/blank caption drop it
print(captions.shape)
captions = captions.dropna()
print(captions.shape)
# training and testing image captions split
train_image_captions = {}
test_image_captions = {}
# list for storing every caption
all_captions = []
# storing training data
for image in train_data_images:
    tempDf = captions[captions['image'] == image]
    list_of_captions = tempDf['caption'].tolist()
    train_image_captions[image] = list_of_captions
    all_captions.append(list_of_captions)
# store testing data
for image in test_data_images:
    tempDf = captions[captions['image'] == image]
    list_of_captions = tempDf['caption'].tolist()
    test_image_captions[image] = list_of_captions
    all_captions.append(list_of_captions)
print("Data Statistics")
print(f"Training Images Captions {len(train_image_captions.keys())}")
print(f"Testing Images Captions {len(test_image_captions.keys())}")

train_image_captions[list(train_image_captions. Keys())[150]]
['<start> A brown dog chases a tattered ball around the yard . <end>',
  '<start> A brown dog is chasing a tattered soccer ball across a low cut field . <end>',
  '<start> Large brown dog playing with a white soccer ball in the grass . <end>',
  '<start> Tan dog chasing a ball . <end>',
  '<start> The tan dog is chasing a ball . <end>']

import spacy
nlp = spacy.load('en', disable=['tagger', 'parser', 'ner'])
# tokenize evry captions, remove punctuations, lowercase everything
for key, value in train_image_captions.items():
    ls = []
    for v in value:
        doc = nlp(v)
        new_v = " "
        for token in doc:
            if not token.is_punct:
                if token.text not in [" ", "\n", "\n\n"]:
                    new_v = new_v + " " + token.text.lower()
        new_v = new_v.strip()
        ls.append(new_v)
    train_image_captions[key] = ls
# create a vocabulary of all the unique words present in captions
# flatten the list
all_captions = [caption for list_of_captions in all_captions for caption in list_of_captions]
# use spacy to convert to lowercase and reject any special characters
tokens = []
for captions in all_captions:
    doc = nlp(captions)
    for token in doc:
        if not token.is_punct:
            if token.text not in [" ", "\n", "\n\n"]:
                tokens.append(token.text.lower())
# get tokens with frequency less than 10
import collections
word_count_dict = collections.Counter(tokens)
reject_words = []
for key, value in word_count_dict.items():
    if value < 10:
        reject_words.append(key)
reject_words.append("<")
reject_words.append(">")
  # remove tokens that are in reject words
tokens = [x for x in tokens if x not in reject_words]
# convert the token to equivalent index using Tokenizer class of Keras
from keras.preprocessing.text import Tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(tokens)