Mark Chapel Mark Chapel - 3 months ago 10
Python Question

Create and write to new csv file during 'for' loop

The code below is intended to create a csv file called 'file-0.csv' and start writing lines by iterating through the for loop until it reaches 100 lines. When the limit is reached, it should stop writing to 'file-0.csv', create 'file-1.csv', and continuing the for loop where it left off, start writing to 'file-1.csv' until it reaches 100 lines, and so on until the for loop is complete.

The actual behavior of the code below (complete, and executable) is that it creates the new files as expected (4 total), but it continues to write all lines to 'file-0'....

##### Python 3.5 #####

import csv

rowCounter = 0
fileCounter = 0
List_A = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C = ['a', 'b', 'c', 'd', 'e', 'f', 'g']


def new_file():
global fileCounter
fileCounter += 1
with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

for word1 in List_A:
for word2 in List_B:
for word3 in List_C:
sentence = word1 + word2 + word3
rowWriter.writerow ([sentence])

rowCounter += 1

if rowCounter == 100:
new_file()
rowCounter = 0
else:
continue


Same code as above, but heavily commented:

##### Python 3.5 #####

######################
####### Setup ########
######################

### Use the CSV library
import csv

### Initialize counters
rowCounter = 0
fileCounter = 0

### Create three lists of 'words'
List_A = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

### Function/subroutine that creates new CSV file with incremented filename
def new_file():
### Make the variable 'fileCounter' usable by the function
global fileCounter
### Add 1 to 'fileCounter'
fileCounter += 1
### Create new CSV file using the value of 'fileCounter' as part of the name
with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)


######################
#### Main Program ####
######################

### Create initial CSV file using the value of 'fileCounter' as part of the name
with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
### Create writer object and define how it should behave
rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

### Create & Write lines ###
### Nested 'for' loops to iterate through all combinations of words
for word1 in List_A:
for word2 in List_B:
for word3 in List_C:
### Build our 'sentence' from the current iteration
sentence = word1 + word2 + word3
### Write 'sentence' to file
rowWriter.writerow ([sentence])

### Increment row counter
rowCounter += 1

### Check if value of rowCounter is 100 and if so, execute
### 'new_file' and reset rowCounter to 0. If not, continue.
if rowCounter == 100:
new_file()
rowCounter = 0
else:
continue


I suspect the problem is 'rowWriter' not getting updated or passed back to the main loop properly, but I can't seem to figure out how to do it (and anyway, I'm not even sure if that's it).

I've tried to document and make the code "generic" so others can get some use out of any answers. Any help is greatly appreciated.

Answer

Leaving the with block closes with file. Therefore, the new_file function just opens and immediately closes a file.

You could do somthing like the following:

import csv

rowCounter     = 0
fileCounter    = 0
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

# create file handle
csvfile = open('file-' + str(fileCounter) + '.csv', 'w')

rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

for word1 in List_A:
    for word2 in List_B:
        for word3 in List_C:
            sentence = word1 + word2 + word3
            rowWriter.writerow ([sentence])

            rowCounter += 1

            if rowCounter == 100:
                # close current filehandle
                csvfile.close()
                fileCounter += 1
                # open new file
                csvfile =  open('file-' + str(fileCounter) + '.csv', 'w')
                rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)
                rowCounter = 0

# close file
csvfile.close()

or with defining a function:

import csv

rowCounter     = 0
fileCounter    = 0
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

def new_writer( csvfile, counter ):
     if csvfile: 
         csvfile.close()

     # open new file
     csvfile =  open('file-' + str(counter) + '.csv', 'w')
     rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

     counter += 1

     return rowWriter,csvfile,counter

rowWriter, csvFile, fileCounter = new_writer( None, fileCounter )

for word1 in List_A:
    for word2 in List_B:
        for word3 in List_C:
            sentence = word1 + word2 + word3
            rowWriter.writerow ([sentence])

            rowCounter += 1

            if rowCounter == 100:
                # close current file and open a new one
                rowWriter, csvfile, counter =  new_writer( csvfile, fileCounter )         
                rowCounter = 0

# close file
csvFile.close()