[3392] | 1 | #Copyright (c) 2015 Raytheon BBN Technologies |
---|
| 2 | # |
---|
| 3 | #Permission is hereby granted, free of charge, to any person obtaining |
---|
| 4 | #a copy of this software and/or hardware specification (the "Work") to |
---|
| 5 | #deal in the Work without restriction, including without limitation the |
---|
| 6 | #rights to use, copy, modify, merge, publish, distribute, sublicense, |
---|
| 7 | #and/or sell copies of the Work, and to permit persons to whom the Work |
---|
| 8 | #is furnished to do so, subject to the following conditions: |
---|
| 9 | # |
---|
| 10 | #The above copyright notice and this permission notice shall be |
---|
| 11 | #included in all copies or substantial portions of the Work. |
---|
| 12 | # |
---|
| 13 | #THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
---|
| 14 | #OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
---|
| 15 | #MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
---|
| 16 | #NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT |
---|
| 17 | #HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
---|
| 18 | #WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
---|
| 19 | #OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS |
---|
| 20 | #IN THE WORK. |
---|
| 21 | |
---|
| 22 | --- |
---|
| 23 | - name: test all hosts |
---|
| 24 | hosts: all |
---|
| 25 | sudo: True |
---|
| 26 | tasks: |
---|
| 27 | - name: neuca test |
---|
| 28 | shell: neuca-user-data |
---|
| 29 | register: neuca_test |
---|
| 30 | - debug: var=neuca_test.stdout_lines |
---|
| 31 | |
---|
| 32 | - name: test if the postboot script is there |
---|
| 33 | shell: if [ -f /home/hadoop/hadoop_config_dynamic.sh ] ; then echo "yes" ; else echo "no" ; fi |
---|
| 34 | register: file_test |
---|
| 35 | failed_when: "'no' in file_test.stdout_lines" |
---|
| 36 | - debug: var=file_test.stdout_lines |
---|
| 37 | |
---|
| 38 | - name: install script has run |
---|
| 39 | shell: if grep -q "Hello from neuca script" "/home/hadoop/hadoop_boot.log" ; then echo "yes" ; else echo "no" ; fi |
---|
| 40 | register: install_test |
---|
| 41 | failed_when: "'no' in install_test.stdout_lines" |
---|
| 42 | - debug: var=install_test.stdout_lines |
---|
| 43 | |
---|
| 44 | - name: test workers |
---|
| 45 | hosts: worker-* |
---|
| 46 | sudo: True |
---|
| 47 | tasks: |
---|
| 48 | - name: ping test |
---|
| 49 | command: ping -c 5 master |
---|
| 50 | register: ping_test |
---|
| 51 | - debug: var=ping_test.stdout_lines |
---|
| 52 | |
---|
| 53 | - name: test master |
---|
| 54 | hosts: master |
---|
| 55 | sudo: True |
---|
| 56 | tasks: |
---|
| 57 | |
---|
| 58 | - name: check hadoop user is present |
---|
| 59 | shell: "cut -d: -f1 /etc/passwd" |
---|
| 60 | register: user_test |
---|
| 61 | failed_when: "'hadoop' not in user_test.stdout" |
---|
| 62 | - debug: var=user_test.stdout |
---|
| 63 | |
---|
| 64 | - name: check formating the hadoop filesystem |
---|
| 65 | become: True |
---|
| 66 | become_user: hadoop |
---|
| 67 | shell: cd ~; source ~/.bashrc; hdfs namenode -format -force |
---|
| 68 | register: hdfs_format |
---|
| 69 | failed_when: "'SHUTDOWN_MSG: Shutting down NameNode at master/172.16.1.1' not in hdfs_format.stderr" |
---|
| 70 | async: 600 |
---|
| 71 | poll: 30 |
---|
| 72 | - debug: var=hdfs_format.stderr |
---|
| 73 | |
---|
| 74 | - name: start dfs |
---|
| 75 | become: True |
---|
| 76 | become_user: hadoop |
---|
| 77 | shell: cd ~; source ~/.bashrc; start-dfs.sh |
---|
| 78 | register: dfs_start |
---|
| 79 | async: 600 |
---|
| 80 | poll: 10 |
---|
| 81 | - debug: var=dfs_start.stderr |
---|
| 82 | |
---|
| 83 | - name: start yarn |
---|
| 84 | become: True |
---|
| 85 | become_user: hadoop |
---|
| 86 | shell: cd ~; source ~/.bashrc; start-yarn.sh |
---|
| 87 | register: yarn_start |
---|
| 88 | async: 600 |
---|
| 89 | poll: 10 |
---|
| 90 | - debug: var=yarn_start.stdout |
---|
| 91 | - debug: var=yarn_start.stderr |
---|
| 92 | |
---|
| 93 | - name: hdfs report |
---|
| 94 | become: True |
---|
| 95 | become_user: hadoop |
---|
| 96 | shell: cd ~; source ~/.bashrc; hdfs dfsadmin -report |
---|
| 97 | register: hdfs_report |
---|
| 98 | async: 600 |
---|
| 99 | poll: 5 |
---|
| 100 | - debug: var=hdfs_report.stdout |
---|
| 101 | - debug: var=hdfs_report.stderr |
---|
| 102 | |
---|
| 103 | - name: small test |
---|
| 104 | become: True |
---|
| 105 | become_user: hadoop |
---|
| 106 | shell: "cd ~; source ~/.bashrc; echo Hello GENI World > /tmp/hello.txt; hdfs |
---|
| 107 | dfs -put /tmp/hello.txt /hello.txt; hdfs dfs -ls /; hdfs dfs -cat /hello.txt" |
---|
| 108 | register: small_test |
---|
| 109 | async: 600 |
---|
| 110 | poll: 5 |
---|
| 111 | failed_when: "'-rw-r--r-- 2 hadoop supergroup' not in small_test.stdout" |
---|
| 112 | - debug: var=small_test.stdout |
---|
| 113 | - debug: var=small_test.stderr |
---|
| 114 | |
---|
| 115 | # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798 |
---|
| 116 | # XXX When this is does remove timeout from shell hadoop command |
---|
| 117 | - name: generate dataset |
---|
| 118 | become: True |
---|
| 119 | become_user: hadoop |
---|
| 120 | shell: "cd ~; source ~/.bashrc; timeout 600 hadoop jar /home/hadoop/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar teragen 10000000 /input" |
---|
| 121 | # async: 600 |
---|
| 122 | # poll: 20 |
---|
| 123 | register: gen_dataset |
---|
| 124 | failed_when: "'Bytes Written=1000000000' not in gen_dataset.stderr" |
---|
| 125 | - debug: var=gen_dataset.stderr |
---|
| 126 | - debug: var=gen_dataset.stdout |
---|
| 127 | |
---|
| 128 | # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798 |
---|
| 129 | # XXX When this is does remove timeout from shell hadoop command |
---|
| 130 | - name: sort dataset |
---|
| 131 | become: True |
---|
| 132 | become_user: hadoop |
---|
| 133 | shell: "cd ~; source ~/.bashrc; timeout 600 hadoop jar /home/hadoop/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar terasort /input /output" |
---|
| 134 | register: sort_dataset |
---|
| 135 | # async: 600 |
---|
| 136 | failed_when: "'INFO terasort.TeraSort: done' not in sort_dataset.stderr" |
---|
| 137 | - debug: var=sort_dataset.stderr |
---|
| 138 | |
---|
| 139 | - name: verify files |
---|
| 140 | become: True |
---|
| 141 | become_user: hadoop |
---|
| 142 | shell: "cd ~; source ~/.bashrc; hdfs dfs -ls /input/; hdfs dfs -ls |
---|
| 143 | /output/; hdfs dfs -cat /input/part-m-00000 | head; hdfs dfs -cat /output/part-r-00000 | head" |
---|
| 144 | register: verify_files |
---|
| 145 | async: 600 |
---|
| 146 | poll: 10 |
---|
| 147 | - debug: var=verify_files.stdout |
---|
| 148 | - debug: var=verify_files.stderr |
---|
| 149 | |
---|
| 150 | # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798 |
---|
| 151 | # XXX When this is does remove timeout from shell execution |
---|
| 152 | - name: hexdump |
---|
| 153 | become: True |
---|
| 154 | become_user: hadoop |
---|
| 155 | shell: "cd ~; source ~/.bashrc; timeout 600 hdfs dfs -get /output/part-r-00000 /tmp/part-r-00000; hexdump /tmp/part-r-00000 | head" |
---|
| 156 | register: hexdump |
---|
| 157 | # async: 600 |
---|
| 158 | # poll: 30 |
---|
| 159 | - debug: var=hexdump.stdout |
---|
| 160 | - debug: var=hexdump.stderr |
---|
| 161 | |
---|