1 | #Copyright (c) 2015 Raytheon BBN Technologies |
---|
2 | # |
---|
3 | #Permission is hereby granted, free of charge, to any person obtaining |
---|
4 | #a copy of this software and/or hardware specification (the "Work") to |
---|
5 | #deal in the Work without restriction, including without limitation the |
---|
6 | #rights to use, copy, modify, merge, publish, distribute, sublicense, |
---|
7 | #and/or sell copies of the Work, and to permit persons to whom the Work |
---|
8 | #is furnished to do so, subject to the following conditions: |
---|
9 | # |
---|
10 | #The above copyright notice and this permission notice shall be |
---|
11 | #included in all copies or substantial portions of the Work. |
---|
12 | # |
---|
13 | #THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
---|
14 | #OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
---|
15 | #MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
---|
16 | #NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT |
---|
17 | #HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
---|
18 | #WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
---|
19 | #OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS |
---|
20 | #IN THE WORK. |
---|
21 | |
---|
22 | --- |
---|
23 | - name: test all hosts |
---|
24 | hosts: all |
---|
25 | sudo: True |
---|
26 | tasks: |
---|
27 | - name: neuca test |
---|
28 | shell: neuca-user-data |
---|
29 | register: neuca_test |
---|
30 | - debug: var=neuca_test.stdout_lines |
---|
31 | |
---|
32 | - name: test if the postboot script is there |
---|
33 | shell: if [ -f /home/hadoop/hadoop_config_dynamic.sh ] ; then echo "yes" ; else echo "no" ; fi |
---|
34 | register: file_test |
---|
35 | failed_when: "'no' in file_test.stdout_lines" |
---|
36 | - debug: var=file_test.stdout_lines |
---|
37 | |
---|
38 | - name: install script has run |
---|
39 | shell: if grep -q "Hello from neuca script" "/home/hadoop/hadoop_boot.log" ; then echo "yes" ; else echo "no" ; fi |
---|
40 | register: install_test |
---|
41 | failed_when: "'no' in install_test.stdout_lines" |
---|
42 | - debug: var=install_test.stdout_lines |
---|
43 | |
---|
44 | - name: test workers |
---|
45 | hosts: worker-* |
---|
46 | sudo: True |
---|
47 | tasks: |
---|
48 | - name: ping test |
---|
49 | command: ping -c 5 master |
---|
50 | register: ping_test |
---|
51 | - debug: var=ping_test.stdout_lines |
---|
52 | |
---|
53 | - name: test master |
---|
54 | hosts: master |
---|
55 | sudo: True |
---|
56 | tasks: |
---|
57 | |
---|
58 | - name: check hadoop user is present |
---|
59 | shell: "cut -d: -f1 /etc/passwd" |
---|
60 | register: user_test |
---|
61 | failed_when: "'hadoop' not in user_test.stdout" |
---|
62 | - debug: var=user_test.stdout |
---|
63 | |
---|
64 | - name: check formating the hadoop filesystem |
---|
65 | become: True |
---|
66 | become_user: hadoop |
---|
67 | shell: cd ~; source ~/.bashrc; hdfs namenode -format -force |
---|
68 | register: hdfs_format |
---|
69 | failed_when: "'SHUTDOWN_MSG: Shutting down NameNode at master/172.16.1.1' not in hdfs_format.stderr" |
---|
70 | async: 600 |
---|
71 | poll: 30 |
---|
72 | - debug: var=hdfs_format.stderr |
---|
73 | |
---|
74 | - name: start dfs |
---|
75 | become: True |
---|
76 | become_user: hadoop |
---|
77 | shell: cd ~; source ~/.bashrc; start-dfs.sh |
---|
78 | register: dfs_start |
---|
79 | async: 600 |
---|
80 | poll: 10 |
---|
81 | - debug: var=dfs_start.stderr |
---|
82 | |
---|
83 | - name: start yarn |
---|
84 | become: True |
---|
85 | become_user: hadoop |
---|
86 | shell: cd ~; source ~/.bashrc; start-yarn.sh |
---|
87 | register: yarn_start |
---|
88 | async: 600 |
---|
89 | poll: 10 |
---|
90 | - debug: var=yarn_start.stdout |
---|
91 | - debug: var=yarn_start.stderr |
---|
92 | |
---|
93 | - name: hdfs report |
---|
94 | become: True |
---|
95 | become_user: hadoop |
---|
96 | shell: cd ~; source ~/.bashrc; hdfs dfsadmin -report |
---|
97 | register: hdfs_report |
---|
98 | async: 600 |
---|
99 | poll: 5 |
---|
100 | - debug: var=hdfs_report.stdout |
---|
101 | - debug: var=hdfs_report.stderr |
---|
102 | |
---|
103 | - name: small test |
---|
104 | become: True |
---|
105 | become_user: hadoop |
---|
106 | shell: "cd ~; source ~/.bashrc; echo Hello GENI World > /tmp/hello.txt; hdfs |
---|
107 | dfs -put /tmp/hello.txt /hello.txt; hdfs dfs -ls /; hdfs dfs -cat /hello.txt" |
---|
108 | register: small_test |
---|
109 | async: 600 |
---|
110 | poll: 5 |
---|
111 | failed_when: "'-rw-r--r-- 2 hadoop supergroup' not in small_test.stdout" |
---|
112 | - debug: var=small_test.stdout |
---|
113 | - debug: var=small_test.stderr |
---|
114 | |
---|
115 | # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798 |
---|
116 | # XXX When this is does remove timeout from shell hadoop command |
---|
117 | - name: generate dataset |
---|
118 | become: True |
---|
119 | become_user: hadoop |
---|
120 | shell: "cd ~; source ~/.bashrc; timeout 600 hadoop jar /home/hadoop/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar teragen 10000000 /input" |
---|
121 | # async: 600 |
---|
122 | # poll: 20 |
---|
123 | register: gen_dataset |
---|
124 | failed_when: "'Bytes Written=1000000000' not in gen_dataset.stderr" |
---|
125 | - debug: var=gen_dataset.stderr |
---|
126 | - debug: var=gen_dataset.stdout |
---|
127 | |
---|
128 | # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798 |
---|
129 | # XXX When this is does remove timeout from shell hadoop command |
---|
130 | - name: sort dataset |
---|
131 | become: True |
---|
132 | become_user: hadoop |
---|
133 | shell: "cd ~; source ~/.bashrc; timeout 600 hadoop jar /home/hadoop/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar terasort /input /output" |
---|
134 | register: sort_dataset |
---|
135 | # async: 600 |
---|
136 | failed_when: "'INFO terasort.TeraSort: done' not in sort_dataset.stderr" |
---|
137 | - debug: var=sort_dataset.stderr |
---|
138 | |
---|
139 | - name: verify files |
---|
140 | become: True |
---|
141 | become_user: hadoop |
---|
142 | shell: "cd ~; source ~/.bashrc; hdfs dfs -ls /input/; hdfs dfs -ls |
---|
143 | /output/; hdfs dfs -cat /input/part-m-00000 | head; hdfs dfs -cat /output/part-r-00000 | head" |
---|
144 | register: verify_files |
---|
145 | async: 600 |
---|
146 | poll: 10 |
---|
147 | - debug: var=verify_files.stdout |
---|
148 | - debug: var=verify_files.stderr |
---|
149 | |
---|
150 | # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798 |
---|
151 | # XXX When this is does remove timeout from shell execution |
---|
152 | - name: hexdump |
---|
153 | become: True |
---|
154 | become_user: hadoop |
---|
155 | shell: "cd ~; source ~/.bashrc; timeout 600 hdfs dfs -get /output/part-r-00000 /tmp/part-r-00000; hexdump /tmp/part-r-00000 | head" |
---|
156 | register: hexdump |
---|
157 | # async: 600 |
---|
158 | # poll: 30 |
---|
159 | - debug: var=hexdump.stdout |
---|
160 | - debug: var=hexdump.stderr |
---|
161 | |
---|