add robust handling of non-UTF8 when detecting inventory/playbooks

awx/main/tests/unit/utils/ansible/inventories/invalid/bad

@@ -0,0 +1 @@
+$$$

awx/main/tests/unit/utils/ansible/inventories/invalid/bad_encoding

@@ -0,0 +1,2 @@
+<EFBFBD>
+?

awx/main/tests/unit/utils/ansible/playbooks/invalid/bad

@@ -0,0 +1 @@
+not a playbook

awx/main/tests/unit/utils/ansible/playbooks/invalid/bad_encoding

@@ -0,0 +1,2 @@
+<EFBFBD>
+?

awx/main/tests/unit/utils/ansible/playbooks/invalid/wrong_file_extension

@@ -0,0 +1,7 @@
+- name: Hello World Sample
+  hosts: all
+  tasks:
+    - name: Hello Message
+      debug:
+        msg: "Hello World!"
+

awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yaml

@@ -0,0 +1,7 @@
+- name: Hello World Sample
+  hosts: all
+  tasks:
+    - name: Hello Message
+      debug:
+        msg: "Hello World!"
+

awx/main/tests/unit/utils/ansible/playbooks/valid/hello_world.yml

@@ -0,0 +1,7 @@
+- name: Hello World Sample
+  hosts: all
+  tasks:
+    - name: Hello Message
+      debug:
+        msg: "Hello World!"
+

awx/main/tests/unit/utils/ansible/playbooks/valid/hosts.yml

@@ -0,0 +1 @@
+- hosts: all

awx/main/tests/unit/utils/ansible/playbooks/valid/import.yml

@@ -0,0 +1 @@
+- import_playbook: foo

awx/main/tests/unit/utils/ansible/playbooks/valid/include.yml

@@ -0,0 +1 @@
+- include: foo

awx/main/tests/unit/utils/ansible/playbooks/valid/vault.yml

@@ -0,0 +1,11 @@
+$ANSIBLE_VAULT;1.1;AES256
+64313966653262616130386430653233326161623364386235636436333430646161323830336663
+6633613437653635626162386338613338646231396363660a376537373331356239623435353365
+31633039363639376166633538336335383062316461633439346630363135316266613766393864
+3363343634636535650a346231653233626362323135383164636634343534333466363139633436
+66366132656364316161336538613933666537666361356662306631653235323936363764613338
+65653833326661323935373535396164373132393165383633643432306432373463376461613165
+61376361323861373036316230343038666366336231303231303937393731616664316664373338
+36623935363262623566653238313964636435666138626336346465363562356535663033356265
+64323239616536346365306635353863623831636266633233313437396236633235373539373363
+6464303039663737333164613763306137393864356663316263

awx/main/tests/unit/utils/ansible/test_ansible.py

@@ -0,0 +1,32 @@
+import os
+import os.path
+
+import pytest
+
+from awx.main.utils.ansible import could_be_playbook, could_be_inventory
+
+HERE, _ = os.path.split(__file__)
+
+
+@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'playbooks', 'valid')))
+def test_could_be_playbook(filename):
+    path = os.path.join(HERE, 'playbooks', 'valid')
+    assert could_be_playbook(HERE, path, filename).endswith(filename)
+
+
+@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'playbooks', 'invalid')))
+def test_is_not_playbook(filename):
+    path = os.path.join(HERE, 'playbooks', 'invalid')
+    assert could_be_playbook(HERE, path, filename) is None
+
+
+@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'inventories', 'valid')))
+def test_could_be_inventory(filename):
+    path = os.path.join(HERE, 'inventories', 'valid')
+    assert could_be_inventory(HERE, path, filename).endswith(filename)
+
+
+@pytest.mark.parametrize('filename', os.listdir(os.path.join(HERE, 'inventories', 'invalid')))
+def test_is_not_inventory(filename):
+    path = os.path.join(HERE, 'inventories', 'invalid')
+    assert could_be_inventory(HERE, path, filename) is None

awx/main/utils/ansible.py

@@ -2,6 +2,7 @@
 # All Rights Reserved.
 
 # Python
+import codecs
 import re
 import os
 from itertools import islice
@@ -44,13 +45,20 @@ def could_be_playbook(project_path, dir_path, filename):
     # show up.
     matched = False
     try:
-        for n, line in enumerate(open(playbook_path)):
+        for n, line in enumerate(codecs.open(
+            playbook_path,
+            'r',
+            encoding='utf-8',
+            errors='ignore'
+        )):
             if valid_playbook_re.match(line):
                 matched = True
+                break
             # Any YAML file can also be encrypted with vault;
             # allow these to be used as the main playbook.
             elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
                 matched = True
+                break
     except IOError:
         return None
     if not matched:
@@ -77,7 +85,12 @@ def could_be_inventory(project_path, dir_path, filename):
     # Ansible inventory mainly
     try:
         # only read through first 10 lines for performance
-        with open(inventory_path) as inv_file:
+        with codecs.open(
+            inventory_path,
+            'r',
+            encoding='utf-8',
+            errors='ignore'
+        ) as inv_file:
             for line in islice(inv_file, 10):
                 if not valid_inventory_re.match(line):
                     return None