Scribd
Upload
6 views4 pages

Text Mining Analysis of App Descriptions

The document discusses text mining of app descriptions from various app categories to analyze term frequencies and associations. It loads app description data from multiple files, preprocesses the text by removing stopwords and punctuation, stems words, and creates a document term matrix (DTM). It then analyzes the DTM to find the most frequent terms, displays them in a barplot and wordcloud, and identifies terms associated with sample words like "time" and "best". It also creates a dummy variable matrix to score descriptions based on presence of popular terms.

Uploaded by

Michael Geography
Copyright
© All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as PDF, TXT or read online on Scribd
6 views4 pages

Text Mining Analysis of App Descriptions

The document discusses text mining of app descriptions from various app categories to analyze term frequencies and associations. It loads app description data from multiple files, preprocesses the text by removing stopwords and punctuation, stems words, and creates a document term matrix (DTM). It then analyzes the DTM to find the most frequent terms, displays them in a barplot and wordcloud, and identifies terms associated with sample words like "time" and "best". It also creates a dummy variable matrix to score descriptions based on presence of popular terms.

Uploaded by

Michael Geography
Copyright
© All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as PDF, TXT or read online on Scribd

C:\Users\User\Google Drive\BC2406 BA I Group 2\Project\Group Project_Text Mining.

R Wednesday, 9 November 2016 11:44 AM


rm(list=ls())

# Prepare a list of all file names #


Apps_data_list <- c("[Link]", "[Link]", "[Link]",
"[Link]", "[Link]")

# Loop through the list of file names to read the files and aggregate into a single dataset
dir <- paste(getwd(), "/", sep = "");
for (x in Apps_data_list){
if (!exists("Apps")) {
Apps_data <- [Link](paste(dir, x, sep = ""));
#Apps_data <- Apps_data[c(1:298), ] # Remove last 2 entry because there are NA
Apps <- Apps_data
}
else {
Apps_data <- [Link](paste(dir, x, sep = ""));
#Apps_data <- Apps_data[c(1:298), ] # Remove last 2 entry because there are NA
Apps <- rbind(Apps, Apps_data)
}
}
head(Apps)
library("NLP")
library("tm")
library("SnowballC")
library("wordcloud")
library("RColorBrewer")
library("stats")

## DATA LOADING ##
View(Apps)
Apps_desc <- Apps[,11]
n_desc <- length(Apps_desc)
n_desc

# Use the First Two Sentences #


# Test with the First Description #
Test_sent <- unlist(strsplit([Link](Apps_desc[1]), split = "[.!?]+")) # Split a document
into sentences
Test_sent
Test_sent[1:2]

# Select the first two sentences #


Apps_Sent <- list()

for (i in 1:n_desc) {
temp <- unlist(strsplit([Link](Apps_desc[i]), split = "[.!?]+")) #Split a document into
sentences
Apps_Sent[[i]] <- temp[1:2] # Use the first 2 sentences
}
Apps_Sent[[1]]
Apps_Sent[[2]]

## Convert Vector/List to Corpus ##


Apps <- Corpus(VectorSource(Apps_Sent))
Apps
Apps[[1]]
[Link](Apps[[1]])

# Upper-Case letters to Lower-Case Letters #


Apps <- tm_map(Apps, content_transformer(tolower))
[Link](Apps[[1]])

## Step 3: Parsing ##
# Delete HTML Tags #
-1-
C:\Users\User\Google Drive\BC2406 BA I Group 2\Project\Group Project_Text Mining.R Wednesday, 9 November 2016 11:44 AM
for (j in 1:n_desc) Apps[[j]] <- gsub("u2019", " ", Apps[[j]]) # Delete "u2019"
for (j in 1:n_desc) Apps[[j]] <- gsub("u'", " ", Apps[[j]]) # Delete u'
for (j in 1:n_desc) Apps[[j]] <- gsub("u\"", " ", Apps[[j]]) #Delete u"
[Link](Apps[[1]])

inspect(Apps[1:3])

# Delete Additional HTML operators


for (j in 1:n_desc) Apps[[j]] <- gsub("u2605", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2606", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u201c", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u201d", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2011", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2013", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2014", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2022", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2122", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2026", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2028", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u2729", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("u20ac", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("amp", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("xae", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("xa0", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("xa3", " ", Apps[[j]])
inspect(Apps[1:3])

# Delete device-related terms: Frequently appeared but less-informative #


for (j in 1:n_desc) Apps[[j]] <- gsub("apple", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("iphone", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("touch", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("ipod", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("ipad", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("3gs", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("3rd", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("2nd", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("4th", " ", Apps[[j]])
inspect(Apps[1:3])

# Delete App-Store Related terms: Frequently appeared but less-informative #


for (j in 1:n_desc) Apps[[j]] <- gsub("app", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("store", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("game", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("play", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("mobile", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("free", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("new", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("world", " ", Apps[[j]])
inspect(Apps[1:3])

# Convert important & meaningful numbers to characters (terms) #


for (j in 1:n_desc) Apps[[j]] <- gsub("#1", "numberone", Apps[[j]]) #1
for (j in 1:n_desc) Apps[[j]] <- gsub("99", "nintyninecent", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("%", "percent", Apps[[j]])

# Remove stopwords #
Apps <- tm_map(Apps, removeWords, stopwords("english"))
inspect(Apps[1:3])
newstopwords <- c("and", "for", "the", "to", "in", "when", "then", "he", "she", "than", "can",
"get", "one");
Apps <- tm_map(Apps, removeWords, newstopwords)
for (j in 1:n_desc) Apps[[j]] <- gsub("don", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("won", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("ing", " ", Apps[[j]])
-2-
C:\Users\User\Google Drive\BC2406 BA I Group 2\Project\Group Project_Text Mining.R Wednesday, 9 November 2016 11:44 AM
for (j in 1:n_desc) Apps[[j]] <- gsub("http", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("'ll", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("www", " ", Apps[[j]])
for (j in 1:n_desc) Apps[[j]] <- gsub("com", " ", Apps[[j]])
inspect(Apps[1:3])

# Remove Numbers #
Apps <- tm_map(Apps, removeNumbers)
inspect(Apps[1:3])

# Remove punctuations and symbols #


Apps <- tm_map(Apps, removePunctuation)
inspect(Apps[1:3])

# Manually Delete Non-Characters #


for (j in 1:n_desc) Apps[[j]] <- gsub("['â|*|&|-|/|\\|()|\\.,!-_]", " ", Apps[[j]])

# Remove White space #


Apps <- tm_map(Apps, stripWhitespace)
inspect(Apps[1:3])

## STEP 3: STEMMING ##
Apps <- tm_map(Apps, PlainTextDocument)
Apps <- tm_map(Apps, stemDocument)

[Link](Apps[[1]])
[Link](Apps[[3]])

# Original DTM #
dtm_Apps <- DocumentTermMatrix(Apps)
dtm_Apps
#inspect(dtm_Apps)
[Link](Apps[[1]])

dtm_Apps_Ctrl <- DocumentTermMatrix(Apps, control=list(wordLength=c(3,20),


bounds=list(global=c(50,1000))))
dtm_Apps_Ctrl
#inspect(dtm_Apps_Ctrl) # Display DTM for the Descriptions
[Link](Apps[[1]])

inspect(dtm_Apps_Ctrl[1:5,1:15]) #DTM for the first 5 descriptions with the first 15 names

## STEP 4: EVALUATE THE OUTCOMES ##

## TASK 1: Find the terms which occur at least 150 times ##

findFreqTerms(dtm_Apps_Ctrl, 150)
Freq_term <-colSums([Link](dtm_Apps_Ctrl))
Order_Freq_term <- order(Freq_term, decreasing = TRUE)
Freq_term[Order_Freq_term]

.# Frequency Diagram #
library(grDevices)
Apps_DTM_DF = [Link]([Link](dtm_Apps_Ctrl))
numwords <- 20 #The most frequent 20 terms

# Sum each column and sory by descending order #


Terms_Freq <- [Link](sort(sapply(Apps_DTM_DF, FUN=sum), decreasing=TRUE) [1:numwords],
colnames=count)
x <- sort(Terms_Freq[1:numwords,], decreasing=FALSE)
barplot(x, horiz=TRUE, [Link]=0.5, space=1, las=1, col=[Link](10), main="Frequency of
Terms")

# For Original DTM #


-3-
C:\Users\User\Google Drive\BC2406 BA I Group 2\Project\Group Project_Text Mining.R Wednesday, 9 November 2016 11:44 AM
[Link](2406)
m <- [Link](t(dtm_Apps)) #Convert to matrix
v <- sort(rowSums(m), decreasing=TRUE) # Sort the terms in descending order
w <- [Link](word=names(v), freq=v) #Create data frame indicating name & frequency of terms

WC_Color <- [Link](8, "Set2")


wordcloud(w$word,w$freq, scale=c(3,.1),[Link]=1, [Link]=200, [Link]=F, [Link]=.3,
colors=WC_Color)

# For DTM with Controls #


[Link]()
[Link](2406)
m <- [Link](t(dtm_Apps_Ctrl))
v <- sort(rowSums(m), decreasing=TRUE)
w <- [Link](word=names(v), freq=v)

WC_Color <- [Link](8, "Set2")


wordcloud(w$word,w$freq, scale=c(3,.1),[Link]=1, [Link]=200, [Link]=F, [Link]=.3,
colors=WC_Color)

# Find Associated Terms #


findAssocs(dtm_Apps_Ctrl, "time", .3)
findAssocs(dtm_Apps_Ctrl, "now", .2)
findAssocs(dtm_Apps_Ctrl, "best", .2)
findAssocs(dtm_Apps_Ctrl, "featur", .2)
findAssocs(dtm_Apps_Ctrl, "friend", .2)

# Set up new dummy variable for Description #


terms_dummy <- matrix(data = 0, n_desc, numwords);
# Setting the value in dummy variable
for (j in 1:n_desc) {
sentences <- [Link](Apps[[j]]);
for (i in 1:numwords) {
for (m in sentences) {
if (grepl(names(x)[i], m, fixed = TRUE)) {
# if term is found, set 1 and break
terms_dummy[j, i] = 1;
break;
}
}
}
}

# Caculating the score for Popular Terms


terms_score <- [Link](rowSums(terms_dummy));

-4-

You might also like