p.1/22

Size: px

Start display at page:

Download "p.1/22"

かずただのえ
4 years ago
Views:

1 p.1/22

2 & & & & Excel / p.2/22

3 & & & & Excel / p.2/22

4 ( ) ( ) p.3/22

5 ( ) ( ) Baldi Web p.3/22

6 ( ) ( ) Baldi Web ( ) ( ) ( p.3/22 )

7 Text Mining for Clementine True Teller Text Mining Studio Text Miner Trustia p.4/22

8 Text Mining for Clementine True Teller Text Mining Studio Text Miner Trustia : p.4/22

9 Text Mining for Clementine True Teller Text Mining Studio Text Miner Trustia : R p.4/22

10 Text Mining for Clementine True Teller Text Mining Studio Text Miner Trustia : R p.4/22

11 2007 ( ) p.5/22

12 2007 ( ) p.5/22

13 3 (1) (2) (3) (2) (3) p.6/22

14 17 (E) (W) M F (2) Q3A2 4(2) Q4A (3) 4(3) Q3A3, Q4A3 p.7/22

15 R xtabs(~q3a3 + Q4A3 ) Q4A3 ( ) H( ) < > L( ) Q3A3 ( ) H1 H2 M3 L4 L5 H H M L L p.8/22

16 R xtabs(~q3a3 + Q4A3 ) Q4A3 ( ) H( ) < > L( ) Q3A3 ( ) H1 H2 M3 L4 L5 H H M L L R mcnemar.test (xtabs(~q3a3 + Q4A3)) McNemar s chi-squared = , df = 10, p-value = 1.768e-08 p.8/22

17 ID EW Age Sex Q3A3 7 W 20 F M3 17 W 20 F H2 27 E 20 F M3 32 W 10 M M3 37 W 10 F M3 p.9/22

18 tm (tm_0.2-3 by Ingo Feinerer) (HTML,XML,Gmane,RSS) stopwords (and, or,...) ( 13 ) stemming (study -> studies, studied, studying) (11 ) (tf-idf ) p.10/22

19 tm (tm_0.2-3 by Ingo Feinerer) (HTML,XML,Gmane,RSS) stopwords (and, or,...) ( 13 ) stemming (study -> studies, studied, studying) (11 ) (tf-idf ) library(tm) # Text Mining file.obj <- "/Target/textDir" # Alice was beginning to get very tired of sitting by her sister on the bank,.. alice.dc <- TextDocCol(DirSource(file.obj), readercontrol = list(reader = readplain, language = en_us, load = TRUE )) alice.dc # show(alice.dc) A text document collection with 1 text document # stopwords alice.dc3 <- tmmap(alice.dc2, removewords, stopwords("english")) # Stemming alice.dc4 <- tmmap(alice.dc3, stemdoc) inspect(alice.dc4) # "alic" "was" "begin" "to" "get" "veri" "tire" "of" "sit" "by" "her" "sister" "on" "the" bank... p.10/22

20 $ mecab # p.11/22

21 $ mecab #,*,*,*,*,*,,, EOS,,*,*,*,*,,,,,*,*,*,*,,,,,*,*,,,,,,,*,*,*,*,,,,,*,*,,,,,,*,*,*,,,,,,*,*,*,,,,,,,*,*,*,*,,,,,*,*,*,*,,, p.11/22

22 R R C C p.12/22

$h> SEXP mecab3(sexp aa){ SEXP parsed; const char* input = CHAR(STRING_ELT(aa,0)); mecab = mecab_new2 (input); CHECK(mecab); result = mecab_sparse_tostr(mecab, input); CHECK(result); PROTECT(parsed =$

23 R R C C # RMeCab? #include <R.h> #include <Rdefines.h> #include <mecab.h> #include <stdio.h> SEXP mecab3(sexp aa){ SEXP parsed; const char* input = CHAR(STRING_ELT(aa,0)); mecab = mecab_new2 (input); CHECK(mecab); result = mecab_sparse_tostr(mecab, input); CHECK(result); PROTECT(parsed = mkstring(result)); UNPROTECT(1); mecab_destroy(mecab); return(parsed); } p.12/22

24 RMeCab res <-.Call ("mymecab", ) " " " " " " " " " " " " " " p.13/22

25 RMeCab res <-.Call ("mymecab", ) " " " " " " " " " " " " " " res[names(res) == " "] " " " " p.13/22

26 RMeCab res <-.Call ("mymecab", ) " " " " " " " " " " " " " " res[names(res) == " "] " " " " length(res) 7 7 ( ) p.13/22

27 wilcox.test(q3length, Q4length, paired = TRUE) V = 1239, p-value = 5.327e-08 p.14/22

28 wilcox.test(q3length, Q4length, paired = TRUE) V = 1239, p-value = 5.327e-08 (GLM) 1 (EW) 2 (Sex) 3 (Q3A3, Q4A3) p.14/22

29 wilcox.test(q3length, Q4length, paired = TRUE) V = 1239, p-value = 5.327e-08 (GLM) 1 (EW) 2 (Sex) 3 (Q3A3, Q4A3) ( ) QA <- ordered (Q3A3, labels = c("l5", "L4", "H3", "H2", "H1")) Levels: L5 < L4 < H3 < H2 < H1 p.14/22

30 GLM 158 A, B A 3(2)-(3) B 4(2)-(3) p.15/22

31 GLM 158 A, B A 3(2)-(3) B 4(2)-(3) : glm(length ~EW + Sex + QA, family = "poisson",data = sample12) p.15/22

32 GLM 158 A, B A 3(2)-(3) B 4(2)-(3) : glm(length ~EW + Sex + QA, family = "poisson",data = sample12) glm(length ~EW + Sex + QA + pers ( ), family = "poisson") Estimate Std. Error z value Pr(> z ) (Intercept) <2e-16 *** EWW SexF QA.L QA.Q QA.C QAˆ persc * Null deviance: on 158 degrees of freedom Residual deviance: on 151 degrees of freedom p.15/22

33 levels(gobi) " " " " "( ) " " " "( ) " " " "( ) " " " " " " " " " "( ) " "( ) " " " " " " " " " " " " " " " " " " " " " "( ) " "( ) " " " " " " " " " " " p.16/22

34 levels(gobi) " " " " "( ) " " " "( ) " " " "( ) " " " " " " " " " "( ) " "( ) " " " " " " " " " " " " " " " " " " " " " "( ) " "( ) " " " " " " " " " " " p.16/22

35 levels(gobi) " " " " "( ) " " " "( ) " " " "( ) " " " " " " " " " "( ) " "( ) " " " " " " " " " " " " " " " " " " " " " "( ) " "( ) " " " " " " " " " " " p.16/22

36 p.17/22

37 p.17/22

38 data[gobi == " ", "gobi"] <- " " # data[gobi == " ", "gobi"] <- " " data[gobi == " ", "gobi"] <- " " levels(gobi) " " " " "( ) " "( ) " " " " " " " " " " " " " " " " " " " "" " " " " "( ) " " " " " " " p.17/22

39 :H1( ),H2( ),M1( ),L4( ),L5( ) : p.18/22

40 :H1( ),H2( ),M1( ),L4( ),L5( ) : : ( ) ( ) -> : ( ) -> p.18/22

41 dat.t1 <- ftable(xtabs(~ew + Sex + Q3A3 + gobi, data = dat)) dat.t2 <- dat.t1 [row- Sums(dat.t1)!= 0, ] dat.corr <- corresp (dat.t2, nf = min(nrow(dat.t2), ncol(dat.t2) )) biplot(dat.corr) p.19/22

42 dat.t1 <- ftable(xtabs(~ew + Sex + Q3A3 + gobi, data = dat)) dat.t2 <- dat.t1 [row- Sums(dat.t1)!= 0, ] dat.corr <- corresp (dat.t2, nf = min(nrow(dat.t2), ncol(dat.t2) )) biplot(dat.corr) p.19/22

43 1 p.20/22

44 1 EMM3 ( : : ) EFM3 ( : : ) WFH2 ( : : ) EFH2 ( : : ) EFL4 ( : : ) p.20/22

45 2 " " " " " " " " " " " " " " " " p.21/22

46 2 " " " " " " " " " " " " " " " " # library(nnet) model1 <- multinom(gobi ~EW + Sex + Q4A3) model2 <- step (model1) summary(model2) # # multinom(formula = gobi ~EW) p.21/22

47 2 " " " " " " " " " " " " " " " " # library(nnet) model1 <- multinom(gobi ~EW + Sex + Q4A3) model2 <- step (model1) summary(model2) # # multinom(formula = gobi ~EW) # ( ) glm(gobi ~EW + Sex + Q4A3, family = binomial) Estimate z value Pr(> z ) EWW * p.21/22

48 A Language and Environment for Statistical Computing: R, Taku Kudo : MeCab, M. Konchady: Text Mining Application,2006 J.J. Faraway: Extending the Linear Model with R, 2005 Manning & Schuetze: Foundations of Statistical Natural Language Processing, 1999 Lebart & Salem & Berry: Exploring Textual Data, 1998 p.22/22

R Java, C, Perl R R p.2/17

R R p.1/17 R Java, C, Perl R R p.2/17 R Java, C, Perl R R R p.2/17 R lsa tm String Kernel R p.3/17 R lsa tm String Kernel R gsubfn R p.3/17 (XML ) R p.4/17 (XML ) stopwords "a" "about" "above" "across"