[object-reconstructor]
run_pause = 30
concurrency = 1
stats_interval = 30
http_timeout = 60
node_timeout = 10
lockup_timeout = 1800
reclaim_age = 604800
ring_check_interval = 15
handoffs_first = true
Jun 11 07:38:30 localhost.localdomain object-server: 192.168.11.12 - - [11/Jun/2015:07:38:30 +0000] "GET /d54/88/AUTH_test/ssbench-ectest-tiny-writeonly-0604_183408_000092_ec64-isolation/objects_000093" 200 110341 "-" "-" "-" 0.0018 "-" 27269 3
Jun 11 07:38:30 localhost.localdomain object-server: 192.168.11.12 - - [11/Jun/2015:07:38:30 +0000] "GET /d55/88/AUTH_test/ssbench-ectest-tiny-writeonly-0604_183408_000092_ec64-isolation/objects_000093" 404 - "-" "-" "-" 0.0009 "-" 27263 3
Jun 11 07:38:30 localhost.localdomain object-server: 192.168.11.12 - - [11/Jun/2015:07:38:30 +0000] "GET /d55/88/AUTH_test/ssbench-ectest-miniscule-writeonly-0604_183408_000096_ec64-isolation/objects_001088" 404 - "-" "-" "-" 0.0010 "-" 27269 3
Jun 11 07:38:30 localhost.localdomain object-server: 192.168.11.12 - - [11/Jun/2015:07:38:30 +0000] "GET /d54/88/AUTH_test/ssbench-ectest-miniscule-writeonly-0604_183408_000096_ec64-isolation/objects_001088" 200 82 "-" "-" "-" 0.0017 "-" 27273 3
Jun 11 07:38:31 localhost.localdomain object-reconstructor: 192.168.12.15:6003/d34/608 EXCEPTION in replication.Sender: #012Traceback (most recent call last):#012 File "/usr/lib/pymodules/python2.7/swift/obj/ssync_sender.py", line 73, in __call__#012 self.missing_check()#012 File "/usr/lib/pymodules/python2.7/swift/obj/ssync_sender.py", line 216, in missing_check#012 self.connection.send('%x\r\n%s\r\n' % (len(msg), msg))#012 File "/usr/lib/python2.7/httplib.py", line 790, in send#012 self.sock.sendall(data)#012 File "/usr/local/lib/python2.7/dist-packages/eventlet/greenio/base.py", line 376, in sendall#012 tail = self.send(data, flags)#012 File "/usr/local/lib/python2.7/dist-packages/eventlet/greenio/base.py", line 359, in send#012 total_sent += fd.send(data[total_sent:], flags)#012error: [Errno 32] Broken pipe
Jun 11 07:38:31 localhost.localdomain account-server: 192.168.12.15 - - [11/Jun/2015:07:38:31 +0000] "HEAD /d53/3583/.expiring_objects" 404 - "HEAD http://localhost/v1/.expiring_objects" "tx69056c9c5836408287297-0055793af7" "proxy-server 560" 0.0008 "-" 4467 -
Jun 11 07:38:31 localhost.localdomain account-server: 192.168.12.15 - - [11/Jun/2015:07:38:31 +0000] "GET /d53/3583/.expiring_objects" 404 - "GET http://localhost/v1/.expiring_objects?format=json&marker=&end_marker=" "txbac81e6c3da041c699392-0055793af7" "proxy-server 560" 0.0009 "-" 4466 -
Jun 11 07:38:33 localhost.localdomain object-server: 192.168.12.13 - - [11/Jun/2015:07:38:33 +0000] "REPLICATE /d56/2028/001-003-008-00a-018-024-031-038-041-065-071-080-08d-09c-09d-0a8-0b5-0be-0c7-0d8-0dc-0e2-0f4-0fe-0ff-10b-10e-11a-11d-123-128-129-13e-148-14f-155-171-173-181-187-190-195-1a8-1bd-1c7-1ce-1d4-1dc-1eb-1ed-1f0-1f5-203-218-234-247-249-25c-25f-274-275-27c-27f-288-294-29a-2c2-2d6-2e4-2f2-2f6-2f8-301-308-312-314-329-32d-331-332-341-349-35a-360-367-36b-36c-39c-39f-3a2-3a4-3a6-3aa-3b7-3b8-3bf-3c4-3c5-3cc-3d1-3dc-3e2-3ef-3f9-3fc-400-40a-41d-422-42e-43a-443-445-446-448-44e-451-458-460-465-477-479-4a1-4a7-4bb-4bd-4c8-4d4-4d8-4ec-4f6-4fc-50b-50f-514-52c-53b-53e-548-55b-55e-564-567-57b-58a-5a0-5a6-5a9-5ad-5ae-5b0-5b8-5c1-5cd-5d7-5e6-5f1-5fb-602-604-617-620-629-62c-634-64a-679-67c-683-697-6a2-6aa-6b3-6b5-6bc-6be-6d7-6e7-6ea-6ec-6fa-712-713-71d-72a-72d-732-734-74f-753-762-76a-770-771-772-77e-78c-791-796-79e-7a7-7b2-7c3-7c5-7d4-7d7-7de-7f2-7f3-800-811-813-821-829-82a-836-85a-861-86a-86b-875-877-87b-881-882-883-89b-8a8-8ad-8c5-8c8-8d4-8e2-8e5-8f3-8f8-8f9-8fc-900-909-91c-920-923-926-929-92d-933-934-93b-940-949-953-95e-966-971-98c-9a3-9a5-9aa-9af-9b1-9b2-9c4-9c6-9d0-9da-9e7-9ff-a0d-a12-a3e-a51-a67-a87-a8e-a96-abf-ac0-ac5-acc-ad1-ad7-ae3-af0-b08-b0a-b0b-b30-b40-b49-b4b-b52-b5b-b5c-b64-b66-b68-b6d-b71-b7a-b86-b87-b97-b98-b9f-ba7-bc6-bcb-be4-bea-bf8-c09-c0c-c22-c26-c28-c2b-c47-c5b-c63-c65-c68-c6a-c6d-c72-c74-c78-c83-c89-c8e-c9f-ca6-cb2-cb7-cbf-cc6-cc7-ccc-cd0-ceb-cec-ced-cf3-cf6-cf7-d05-d08-d0c-d11-d12-d1c-d20-d30-d36-d3b-d3d-d44-d58-d5b-d67-d7a-d8a-d94-d96-d97-dab-dbf-dc5-dc7-dc8-dcc-dcf-de3-de5-df2-dfe-e0e-e15-e1d-e2b-e48-e68-e6c-e7c-e7e-e86-e97-ea5-eb3-eb6-eb9-ed9-ee4-eec-ef3-f05-f19-f33-f3a-f5e-f60-f69-f79-f7f-f83-f89-f8f-f97-f99-f9a-fb0-fb4-fb8-fc9-fcb-fd0-fd7-fe1-fea-fec-ff0-ffb" 200 47872 "-" "-" "obj-reconstructor 8429" 15.2514 "-" 27264 2
Jun 11 07:38:33 localhost.localdomain object-server: 192.168.12.13/d56/2028 EXCEPTION in replication.Receiver: #012Traceback (most recent call last):#012 File "/usr/lib/pymodules/python2.7/swift/obj/ssync_receiver.py", line 105, in __call__#012 for data in self.updates():#012 File "/usr/lib/pymodules/python2.7/swift/obj/ssync_receiver.py", line 300, in updates#012 line = self.fp.readline(self.app.network_chunk_size)#012 File "/usr/local/lib/python2.7/dist-packages/eventlet/wsgi.py", line 194, in readline#012 return self._chunked_read(self.rfile, size, True)#012 File "/usr/local/lib/python2.7/dist-packages/eventlet/wsgi.py", line 179, in _chunked_read#012 self.chunk_length = int(rfile.readline().split(b";", 1)[0], 16)#012ValueError: invalid literal for int() with base 16: ''
Jun 11 07:38:34 localhost.localdomain account-server: 192.168.12.11 - - [11/Jun/2015:07:38:34 +0000] "HEAD /d48/3746/.misplaced_objects" 404 - "HEAD http://localhost/v1/.misplaced_objects" "tx07ac34bb9cdd4e2581d60-0055793afa" "proxy-server 25237" 0.0008 "-" 4466 -
Jun 11 07:38:34 localhost.localdomain account-server: 192.168.12.12 - - [11/Jun/2015:07:38:34 +0000] "HEAD /d48/3746/.misplaced_objects" 404 - "HEAD http://localhost/v1/.misplaced_objects" "tx48432c439bdb44a6a4d56-0055793afa" "proxy-server 22757" 0.0008 "-" 4466 -
Here we have a few unrelated requests (even ignoring the misplaced objects requests, and the long debug replicate request will all the suffixes in the path)
object- reconstructor: 192.168. 12.15:6003/ d34/608 Exception Broken Pipe <- that's ssync saying the remote end closed it's connection on him unexpectedly
object-server: 192.168. 12.13/d56/ 2028 Exception invalid literal for int() <- that's the object server saying that ssnc closed it's connection on him unexpectedly
Both of this exceptions could be very interesting but only if we can see the other side.
If we can't track down the one or the other of them that's not a lot we can do to debug either of these - and we'll have to close this one as invalid - unless we want to maybe just change the description to "when ssync or the object-server get disconnected it should warn with a more ascetically pleasing message instead of a traceback"
... which wouldn't be an altogether bad idea.