def filename(name, index)
  unless index[name]
    index[name] = true
    return name
  end

  i = 1
  loop do
    new_name = "#{name}(#{i})"
    unless index[new_name]
      index[new_name] = true
      return new_name
    end
    i += 1
  end
end

def filenames(names)
  index = {}

  names.map do |name|
    filename(name, index)
  end
end

names = ["photo", "photo(1)", "photo", "photo(1)", "photo"] # photo, photo(1), photo(2), photo(1)(1), photo(3)
p filenames(names)